Introduce OpenTelemetry (OTEL) instrumentation (#3163)

Resolves #3005

This PR instruments The Combine using OpenTelemetry for observability via the following changes:

* Adds OpenTelemetry instrumentation for tracing.
* Creates a LocationProvider that extracts the IP address from the http context and returns the IP’s location. This location information is added as tags to every trace.
* Adds the session ID as a tag to every trace to group events occurring during the same session.
* Starts new activities with custom trace tags in word-related functions to demonstrate more granular tracing.
* Installs [OpenTelemetry Collector Helm Chart](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) in Kubernetes cluster and adds custom configuration to send traces to Honeycomb and prepare foundation for additional data handling.
* Sets the OTEL_SERVICE_NAME during Kubernetes setup to select Honeycomb dataset according to the host on which The Combine is installed.

Additional changes in this PR:

* Fixes Kubernetes setup to properly interpret branch names containing '/' characters.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/sillsdev/TheCombine/3163)
<!-- Reviewable:end -->
---------

Co-authored-by: Jim Grady <[email protected]>
Co-authored-by: Danny Rorabaugh <[email protected]>

Backend.Tests/Backend.Tests.csproj

@@ -13,6 +13,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="Moq.Contrib.HttpClient" Version="1.4.0" />
     <PackageReference Include="NUnit" Version="4.2.2" />
     <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
     <PackageReference Include="coverlet.collector" Version="6.0.2"/>

Backend.Tests/Mocks/LocationProviderMock.cs

@@ -0,0 +1,20 @@
+using System.Threading.Tasks;
+using BackendFramework.Interfaces;
+using BackendFramework.Otel;
+
+namespace Backend.Tests.Mocks
+{
+    sealed internal class LocationProviderMock : ILocationProvider
+    {
+        public Task<LocationApi?> GetLocation()
+        {
+            LocationApi location = new LocationApi
+            {
+                Country = "test country",
+                RegionName = "test region",
+                City = "city"
+            };
+            return Task.FromResult<LocationApi?>(location);
+        }
+    }
+}

Backend.Tests/Otel/LocationProviderTests.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using BackendFramework.Otel;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.DependencyInjection;
+using Moq;
+using Moq.Protected;
+using NUnit.Framework;
+
+namespace Backend.Tests.Otel
+{
+    public class LocationProviderTests
+    {
+        private readonly IPAddress TestIpAddress = new(new byte[] { 100, 0, 0, 0 });
+        private IHttpContextAccessor _contextAccessor = null!;
+        private IMemoryCache _memoryCache = null!;
+        private Mock<HttpMessageHandler> _handlerMock = null!;
+        private Mock<IHttpClientFactory> _httpClientFactory = null!;
+        private LocationProvider _locationProvider = null!;
+
+        [SetUp]
+        public void Setup()
+        {
+            // Set up HttpContextAccessor with mocked IP
+            _contextAccessor = new HttpContextAccessor();
+            var httpContext = new DefaultHttpContext()
+            {
+                Connection =
+                {
+                    RemoteIpAddress = TestIpAddress
+                }
+            };
+            _contextAccessor.HttpContext = httpContext;
+
+            // Set up MemoryCache
+            var services = new ServiceCollection();
+            services.AddMemoryCache();
+            var serviceProvider = services.BuildServiceProvider();
+            _memoryCache = serviceProvider.GetService<IMemoryCache>()!;
+
+            var result = new HttpResponseMessage()
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent("{}")
+            };
+
+            // Set up HttpClientFactory mock using httpClient with mocked HttpMessageHandler
+            _handlerMock = new Mock<HttpMessageHandler>();
+            _handlerMock.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>()
+            )
+            .ReturnsAsync(result)
+            .Verifiable();
+
+            var httpClient = new HttpClient(_handlerMock.Object);
+            _httpClientFactory = new Mock<IHttpClientFactory>();
+            _httpClientFactory
+                .Setup(x => x.CreateClient(It.IsAny<string>()))
+                .Returns(httpClient);
+
+            _locationProvider = new LocationProvider(_contextAccessor, _memoryCache, _httpClientFactory.Object);
+        }
+
+        public static void Verify(Mock<HttpMessageHandler> mock, Func<HttpRequestMessage, bool> match)
+        {
+            mock.Protected()
+            .Verify(
+                "SendAsync",
+                Times.Exactly(1),
+                ItExpr.Is<HttpRequestMessage>(req => match(req)),
+                ItExpr.IsAny<CancellationToken>()
+            );
+        }
+
+        [Test]
+        public async Task GetLocationHttpClientUsesIp()
+        {
+            // Act
+            await _locationProvider.GetLocationFromIp(TestIpAddress.ToString());
+
+            // Assert
+            Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
+            Verify(_handlerMock, r => !r.RequestUri!.AbsoluteUri.Contains("123.1.2.3"));
+        }
+
+        [Test]
+        public async Task GetLocationUsesHttpContextIp()
+        {
+            // Act
+            await _locationProvider.GetLocation();
+
+            // Assert
+            Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
+            Verify(_handlerMock, r => !r.RequestUri!.AbsoluteUri.Contains("123.1.2.3"));
+        }
+
+        [Test]
+        public async Task GetLocationUsesCache()
+        {
+            // Act
+            // call getLocation twice and verify async method is called only once
+            await _locationProvider.GetLocation();
+            await _locationProvider.GetLocation();
+
+            // Assert
+            Verify(_handlerMock, r => r.RequestUri!.AbsoluteUri.Contains(TestIpAddress.ToString()));
+        }
+    }
+}

Backend.Tests/Otel/OtelKernelTests.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Backend.Tests.Mocks;
+using BackendFramework.Otel;
+using Microsoft.AspNetCore.Http;
+using NUnit.Framework;
+using static BackendFramework.Otel.OtelKernel;
+
+namespace Backend.Tests.Otel
+{
+    public class OtelKernelTests : IDisposable
+    {
+        private const string FrontendSessionIdKey = "sessionId";
+        private const string OtelSessionIdKey = "sessionId";
+        private const string OtelSessionBaggageKey = "sessionBaggage";
+        private LocationEnricher _locationEnricher = null!;
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                _locationEnricher?.Dispose();
+            }
+        }
+
+        [Test]
+        public void BuildersSetSessionBaggageFromHeader()
+        {
+            // Arrange
+            var httpContext = new DefaultHttpContext();
+            httpContext.Request.Headers[FrontendSessionIdKey] = "123";
+            var activity = new Activity("testActivity").Start();
+
+            // Act
+            TrackSession(activity, httpContext.Request);
+
+            // Assert
+            Assert.That(activity.Baggage.Any(_ => _.Key == OtelSessionBaggageKey));
+        }
+
+        [Test]
+        public void OnEndSetsSessionTagFromBaggage()
+        {
+            // Arrange
+            var activity = new Activity("testActivity").Start();
+            activity.SetBaggage(OtelSessionBaggageKey, "test session id");
+
+            // Act
+            _locationEnricher.OnEnd(activity);
+
+            // Assert
+            Assert.That(activity.Tags.Any(_ => _.Key == OtelSessionIdKey));
+        }
+
+
+        [Test]
+        public void OnEndSetsLocationTags()
+        {
+            // Arrange
+            _locationEnricher = new LocationEnricher(new LocationProviderMock());
+            var activity = new Activity("testActivity").Start();
+
+            // Act
+            _locationEnricher.OnEnd(activity);
+
+            // Assert
+            var testLocation = new Dictionary<string, string>
+            {
+                {"country", "test country"},
+                {"regionName", "test region"},
+                {"city", "city"}
+            };
+            Assert.That(activity.Tags, Is.SupersetOf(testLocation));
+        }
+
+        public void OnEndRedactsIp()
+        {
+            // Arrange
+            _locationEnricher = new LocationEnricher(new LocationProviderMock());
+            var activity = new Activity("testActivity").Start();
+            activity.SetTag("url.full", $"{LocationProvider.locationGetterUri}100.0.0.0");
+
+            // Act
+            _locationEnricher.OnEnd(activity);
+
+            // Assert
+            Assert.That(activity.Tags.Any(_ => _.Key == "url.full" && _.Value == ""));
+            Assert.That(activity.Tags.Any(_ => _.Key == "url.redacted.ip" && _.Value == LocationProvider.locationGetterUri));
+        }
+    }
+}

Backend.Tests/Otel/OtelServiceTests.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using BackendFramework.Otel;
+using Microsoft.Extensions.DependencyInjection;
+using NUnit.Framework;
+
+namespace Backend.Tests.Otel
+{
+    public class OtelServiceTests
+    {
+        [Test]
+        public static void TestStartActivityWithTag()
+        {
+            // Arrange
+            var services = new ServiceCollection();
+            services.AddOpenTelemetryInstrumentation();
+            AddActivityListener();
+
+            // Act
+            var activity = OtelService.StartActivityWithTag("test key", "test val");
+            var tag = activity?.GetTagItem("test key");
+            var wrongTag = activity?.GetTagItem("wrong key");
+
+            // Assert
+            Assert.That(activity, Is.Not.Null);
+            Assert.That(tag, Is.Not.Null);
+            Assert.That(wrongTag, Is.Null);
+        }
+
+        private static void AddActivityListener()
+        {
+            var activityListener = new ActivityListener
+            {
+                ShouldListenTo = _ => true,
+                Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllData
+            };
+            ActivitySource.AddActivityListener(activityListener);
+        }
+    }
+}

Backend/BackendFramework.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
-    <LangVersion>10.0</LangVersion>
+    <LangVersion>12.0</LangVersion>
     <Nullable>enable</Nullable>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <AnalysisMode>Recommended</AnalysisMode>
@@ -10,7 +10,13 @@
     <NoWarn>$(NoWarn);CA1305;CA1848;CS1591</NoWarn>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="RelaxNG" Version="3.2.3" >
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
+    <PackageReference Include="RelaxNG" Version="3.2.3">
       <NoWarn>NU1701</NoWarn>
     </PackageReference>
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.3" />