From 4f3281706545bf1efcb9962a3f9617fb43c585b1 Mon Sep 17 00:00:00 2001 From: Brett McBride Date: Wed, 15 May 2024 20:46:42 +1000 Subject: [PATCH] Add infection test tool (#1306) * run unit tests in random order running in random order highlights some interference between tests (mostly logging being enabled), and is a requirement for mutation testing (ie, all tests must pass when executed in a random order). * adding infection mutation testing this adds the ability to run infection. It generates a lot of output, which is an exercise for another PR... * infection max threads --- Makefile | 2 ++ infection.json5 | 19 +++++++++++++++++++ phpunit.xml.dist | 1 + src/API/Behavior/Internal/Logging.php | 6 ++++++ src/API/LoggerHolder.php | 4 ++++ .../Internal/LogWriterFactoryTest.php | 5 +++++ .../API/Behavior/LogsMessagesTraitTest.php | 1 + tests/Unit/API/Common/Time/ClockTest.php | 10 ++++++++++ .../Instrumentation/InstrumentationTest.php | 5 +++++ tests/Unit/API/LoggerHolderTest.php | 5 +++++ .../TraceContextPropagatorTest.php | 5 ++--- tests/Unit/Contrib/Otlp/LogsExporterTest.php | 2 ++ tests/Unit/Contrib/Otlp/SpanExporterTest.php | 2 ++ .../SDK/Metrics/MeterProviderFactoryTest.php | 5 ++--- .../SDK/Metrics/Stream/MetricStreamTest.php | 6 ++++++ .../SDK/Propagation/PropagatorFactoryTest.php | 5 +++-- tests/Unit/SDK/Resource/ResourceInfoTest.php | 5 ++--- tests/Unit/SDK/SdkAutoloaderTest.php | 5 ++--- .../SpanExporter/AbstractExporterTestCase.php | 5 ++--- vendor-bin/infection/composer.json | 10 ++++++++++ 20 files changed, 91 insertions(+), 17 deletions(-) create mode 100644 infection.json5 create mode 100644 vendor-bin/infection/composer.json diff --git a/Makefile b/Makefile index ef34cdce0..df42c09a9 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ psalm-info: ## Run psalm and show info $(DC_RUN_PHP) env XDEBUG_MODE=off vendor-bin/psalm/vendor/bin/psalm --show-info=true --threads=1 phpstan: ## Run phpstan $(DC_RUN_PHP) env XDEBUG_MODE=off vendor/bin/phpstan analyse --memory-limit=256M +infection: ## Run infection (mutation testing) + $(DC_RUN_PHP) env XDEBUG_MODE=coverage php -d memory_limit=1024M vendor-bin/infection/vendor/bin/infection --threads=max packages-composer: ## Validate composer packages $(DC_RUN_PHP) env XDEBUG_MODE=off vendor/bin/otel packages:composer:validate benchmark: ## Run phpbench diff --git a/infection.json5 b/infection.json5 new file mode 100644 index 000000000..eb59bbe6c --- /dev/null +++ b/infection.json5 @@ -0,0 +1,19 @@ +{ + "$schema": "https://raw.githubusercontent.com/infection/infection/0.28.1/resources/schema.json", + "source": { + "directories": [ + "src" + ], + "excludes": [ + "Composer" + ] + }, + "logs": { + "text": "var/infection/infection.log", + "html": "var/infection/infection.html", + }, + "timeout": 1, + "mutators": { + "@default": true + } +} \ No newline at end of file diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 9dca719cc..970b553e7 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -6,6 +6,7 @@ bootstrap="./tests/bootstrap.php" cacheResult="false" colors="false" + executionOrder="random" processIsolation="false" stopOnError="false" stopOnFailure="false" diff --git a/src/API/Behavior/Internal/Logging.php b/src/API/Behavior/Internal/Logging.php index e5bec7ab5..256b5267e 100644 --- a/src/API/Behavior/Internal/Logging.php +++ b/src/API/Behavior/Internal/Logging.php @@ -5,6 +5,7 @@ namespace OpenTelemetry\API\Behavior\Internal; use OpenTelemetry\API\Behavior\Internal\LogWriter\LogWriterInterface; +use OpenTelemetry\API\Behavior\Internal\LogWriter\NoopLogWriter; use Psr\Log\LogLevel; /** @@ -87,4 +88,9 @@ public static function reset(): void self::$logLevel = null; self::$writer = null; } + + public static function disable(): void + { + self::$writer = new NoopLogWriter(); + } } diff --git a/src/API/LoggerHolder.php b/src/API/LoggerHolder.php index 99f916a23..67aa8bfa0 100644 --- a/src/API/LoggerHolder.php +++ b/src/API/LoggerHolder.php @@ -38,6 +38,9 @@ public static function isSet(): bool return null !== self::$logger; } + /** + * @internal + */ public static function unset(): void { self::$logger = null; @@ -45,6 +48,7 @@ public static function unset(): void /** * Disable psr-3 logging + * @internal */ public static function disable(): void { diff --git a/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php b/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php index d5a1107e9..16f6204df 100644 --- a/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php +++ b/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php @@ -21,6 +21,11 @@ class LogWriterFactoryTest extends TestCase { use EnvironmentVariables; + public function setUp(): void + { + LoggerHolder::unset(); + } + public function tearDown(): void { self::restoreEnvironmentVariables(); diff --git a/tests/Unit/API/Behavior/LogsMessagesTraitTest.php b/tests/Unit/API/Behavior/LogsMessagesTraitTest.php index e41d3e01c..08e93f59c 100644 --- a/tests/Unit/API/Behavior/LogsMessagesTraitTest.php +++ b/tests/Unit/API/Behavior/LogsMessagesTraitTest.php @@ -23,6 +23,7 @@ class LogsMessagesTraitTest extends TestCase public function setUp(): void { + Logging::reset(); $this->writer = $this->createMock(LogWriterInterface::class); Logging::setLogWriter($this->writer); } diff --git a/tests/Unit/API/Common/Time/ClockTest.php b/tests/Unit/API/Common/Time/ClockTest.php index 81b2c6c36..1eb622119 100644 --- a/tests/Unit/API/Common/Time/ClockTest.php +++ b/tests/Unit/API/Common/Time/ClockTest.php @@ -14,6 +14,16 @@ */ class ClockTest extends TestCase { + public function setUp(): void + { + Clock::reset(); + } + + public function tearDown(): void + { + Clock::reset(); + } + public function test_default_is_system_clock(): void { $this->assertInstanceOf(SystemClock::class, Clock::getDefault()); diff --git a/tests/Unit/API/Instrumentation/InstrumentationTest.php b/tests/Unit/API/Instrumentation/InstrumentationTest.php index 90e61f310..d0521d73e 100644 --- a/tests/Unit/API/Instrumentation/InstrumentationTest.php +++ b/tests/Unit/API/Instrumentation/InstrumentationTest.php @@ -33,6 +33,11 @@ */ final class InstrumentationTest extends TestCase { + public function setUp(): void + { + Globals::reset(); + } + public function tearDown(): void { Globals::reset(); diff --git a/tests/Unit/API/LoggerHolderTest.php b/tests/Unit/API/LoggerHolderTest.php index 6029c2425..f5beb20e7 100644 --- a/tests/Unit/API/LoggerHolderTest.php +++ b/tests/Unit/API/LoggerHolderTest.php @@ -19,6 +19,11 @@ public function tearDown(): void LoggerHolder::unset(); } + public function setUp(): void + { + LoggerHolder::unset(); + } + public function test_constructor(): void { $logger = $this->createMock(LoggerInterface::class); diff --git a/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php b/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php index 8b4e6859c..aa7ebf1ee 100644 --- a/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php +++ b/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php @@ -4,7 +4,7 @@ namespace OpenTelemetry\Tests\Unit\API\Trace\Propagation; -use OpenTelemetry\API\LoggerHolder; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator; use OpenTelemetry\API\Trace\SpanContext; use OpenTelemetry\API\Trace\SpanContextInterface; @@ -16,7 +16,6 @@ use OpenTelemetry\Context\ContextInterface; use OpenTelemetry\SDK\Trace\Span; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; /** * @covers \OpenTelemetry\API\Trace\Propagation\TraceContextPropagator @@ -35,7 +34,7 @@ class TraceContextPropagatorTest extends TestCase protected function setUp(): void { - LoggerHolder::set(new NullLogger()); + Logging::disable(); $this->traceContextPropagator = TraceContextPropagator::getInstance(); $this->traceState = (new TraceState())->with('bar', 'baz')->with('foo', 'bar'); } diff --git a/tests/Unit/Contrib/Otlp/LogsExporterTest.php b/tests/Unit/Contrib/Otlp/LogsExporterTest.php index fcbc62b8e..9414633b2 100644 --- a/tests/Unit/Contrib/Otlp/LogsExporterTest.php +++ b/tests/Unit/Contrib/Otlp/LogsExporterTest.php @@ -4,6 +4,7 @@ namespace OpenTelemetry\Tests\Unit\Contrib\Otlp; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\Contrib\Otlp\LogsExporter; use OpenTelemetry\SDK\Common\Export\TransportInterface; use OpenTelemetry\SDK\Common\Future\CompletedFuture; @@ -24,6 +25,7 @@ public function setUp(): void $this->transport = $this->createMock(TransportInterface::class); $this->transport->method('contentType')->willReturn('application/x-protobuf'); $this->exporter = new LogsExporter($this->transport); + Logging::disable(); } public function test_export_with_transport_failure(): void diff --git a/tests/Unit/Contrib/Otlp/SpanExporterTest.php b/tests/Unit/Contrib/Otlp/SpanExporterTest.php index c552b80b1..024c5c43d 100644 --- a/tests/Unit/Contrib/Otlp/SpanExporterTest.php +++ b/tests/Unit/Contrib/Otlp/SpanExporterTest.php @@ -5,6 +5,7 @@ namespace OpenTelemetry\Tests\Unit\Contrib\Otlp; use function fseek; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\API\Trace\SpanContext; use OpenTelemetry\Contrib\Otlp\SpanExporter; use OpenTelemetry\SDK\Common\Export\Stream\StreamTransport; @@ -26,6 +27,7 @@ class SpanExporterTest extends TestCase public function setUp(): void { + Logging::disable(); $this->transport = $this->createMock(TransportInterface::class); $this->transport->method('contentType')->willReturn('application/x-protobuf'); $this->exporter = new SpanExporter($this->transport); diff --git a/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php b/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php index c6370bb15..dbc597773 100644 --- a/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php +++ b/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php @@ -5,13 +5,12 @@ namespace OpenTelemetry\Tests\Unit\SDK\Metrics; use AssertWell\PHPUnitGlobalState\EnvironmentVariables; -use OpenTelemetry\API\LoggerHolder; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\API\Metrics\MeterInterface; use OpenTelemetry\SDK\Common\Configuration\KnownValues; use OpenTelemetry\SDK\Common\Configuration\Variables; use OpenTelemetry\SDK\Metrics\MeterProviderFactory; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; /** * @covers \OpenTelemetry\SDK\Metrics\MeterProviderFactory @@ -22,7 +21,7 @@ class MeterProviderFactoryTest extends TestCase public function setUp(): void { - LoggerHolder::set(new NullLogger()); + Logging::disable(); } public function tearDown(): void diff --git a/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php b/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php index 08b4839ba..bf3ae7967 100644 --- a/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php +++ b/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php @@ -6,6 +6,7 @@ use function current; use function extension_loaded; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\Context\Context; use OpenTelemetry\SDK\Common\Attribute\Attributes; use OpenTelemetry\SDK\Metrics\Aggregation\SumAggregation; @@ -44,6 +45,11 @@ */ final class MetricStreamTest extends TestCase { + public function setUp(): void + { + Logging::disable(); + } + public function test_asynchronous_single_data_point(): void { $s = new AsynchronousMetricStream(new SumAggregation(), 3); diff --git a/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php b/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php index 67fdb4f91..8634f375e 100644 --- a/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php +++ b/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php @@ -6,6 +6,7 @@ use AssertWell\PHPUnitGlobalState\EnvironmentVariables; use OpenTelemetry\API\Baggage\Propagation\BaggagePropagator; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\API\LoggerHolder; use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator; use OpenTelemetry\Context\Propagation\MultiTextMapPropagator; @@ -18,7 +19,6 @@ use OpenTelemetry\SDK\Common\Configuration\Variables; use OpenTelemetry\SDK\Propagation\PropagatorFactory; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; /** * @covers \OpenTelemetry\SDK\Propagation\PropagatorFactory @@ -29,7 +29,8 @@ class PropagatorFactoryTest extends TestCase public function setUp(): void { - LoggerHolder::set(new NullLogger()); + LoggerHolder::disable(); + Logging::disable(); } public function tearDown(): void diff --git a/tests/Unit/SDK/Resource/ResourceInfoTest.php b/tests/Unit/SDK/Resource/ResourceInfoTest.php index c20f70b45..ee223472b 100644 --- a/tests/Unit/SDK/Resource/ResourceInfoTest.php +++ b/tests/Unit/SDK/Resource/ResourceInfoTest.php @@ -7,14 +7,13 @@ use AssertWell\PHPUnitGlobalState\EnvironmentVariables; use Composer\InstalledVersions; use Generator; -use OpenTelemetry\API\LoggerHolder; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\SDK\Common\Attribute\Attributes; use OpenTelemetry\SDK\Resource\Detectors; use OpenTelemetry\SDK\Resource\ResourceInfo; use OpenTelemetry\SDK\Resource\ResourceInfoFactory; use OpenTelemetry\SemConv\ResourceAttributes; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; /** * @covers \OpenTelemetry\SDK\Resource\ResourceInfo @@ -25,7 +24,7 @@ class ResourceInfoTest extends TestCase public function setUp(): void { - LoggerHolder::set(new NullLogger()); + Logging::disable(); } public function tearDown(): void diff --git a/tests/Unit/SDK/SdkAutoloaderTest.php b/tests/Unit/SDK/SdkAutoloaderTest.php index e3e4df742..5d59fbe76 100644 --- a/tests/Unit/SDK/SdkAutoloaderTest.php +++ b/tests/Unit/SDK/SdkAutoloaderTest.php @@ -5,8 +5,8 @@ namespace OpenTelemetry\Tests\Unit\SDK; use AssertWell\PHPUnitGlobalState\EnvironmentVariables; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\API\Globals; -use OpenTelemetry\API\LoggerHolder; use OpenTelemetry\API\Logs\NoopEventLoggerProvider; use OpenTelemetry\API\Logs\NoopLoggerProvider; use OpenTelemetry\API\Metrics\Noop\NoopMeterProvider; @@ -15,7 +15,6 @@ use OpenTelemetry\SDK\Common\Configuration\Variables; use OpenTelemetry\SDK\SdkAutoloader; use PHPUnit\Framework\TestCase; -use Psr\Log\NullLogger; /** * @covers \OpenTelemetry\SDK\SdkAutoloader @@ -26,7 +25,7 @@ class SdkAutoloaderTest extends TestCase public function setUp(): void { - LoggerHolder::set(new NullLogger()); + Logging::disable(); Globals::reset(); } diff --git a/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php b/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php index ecc89bcb8..ece1eb14d 100644 --- a/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php +++ b/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php @@ -6,14 +6,13 @@ use Mockery; use Mockery\Adapter\Phpunit\MockeryTestCase; -use OpenTelemetry\API\LoggerHolder; +use OpenTelemetry\API\Behavior\Internal\Logging; use OpenTelemetry\SDK\Common\Export\TransportInterface; use OpenTelemetry\SDK\Common\Future\CompletedFuture; use OpenTelemetry\SDK\Common\Future\ErrorFuture; use OpenTelemetry\SDK\Common\Future\FutureInterface; use OpenTelemetry\SDK\Trace\SpanExporterInterface; use OpenTelemetry\Tests\Unit\SDK\Util\SpanData; -use Psr\Log\NullLogger; /** * @psalm-suppress UndefinedInterfaceMethod @@ -25,7 +24,7 @@ abstract class AbstractExporterTestCase extends MockeryTestCase public function setUp(): void { - LoggerHolder::set(new NullLogger()); + Logging::disable(); $this->future = Mockery::mock(FutureInterface::class); $this->future->allows([ 'map' => $this->future, diff --git a/vendor-bin/infection/composer.json b/vendor-bin/infection/composer.json new file mode 100644 index 000000000..486df3173 --- /dev/null +++ b/vendor-bin/infection/composer.json @@ -0,0 +1,10 @@ +{ + "require-dev": { + "infection/infection": "^0.28.1" + }, + "config": { + "allow-plugins": { + "infection/extension-installer": true + } + } +}