Improve unintelligibility algorithm

OrcanodeMonitor/Core/Fetcher.cs

@@ -788,8 +788,8 @@ public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext con
             try
             {
                 using Stream stream = await _httpClient.GetStreamAsync(newUri);
-                double stdDev = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream);
-                node.AudioStandardDeviation = stdDev;
+                node.AudioStreamStatus = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream);
+                node.AudioStandardDeviation = 0.0;
             } catch (Exception ex)
             {
                 // We couldn't fetch the stream audio so could not update the

OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs

@@ -1,18 +1,73 @@
-using FFMpegCore;
+// Copyright (c) Orcanode Monitor contributors
+// SPDX-License-Identifier: MIT
+using FFMpegCore;
 using FFMpegCore.Pipes;
-using Microsoft.EntityFrameworkCore.SqlServer.Query.Internal;
+using MathNet.Numerics.IntegralTransforms;
 using NAudio.Wave;
+using OrcanodeMonitor.Models;
+using System.Numerics;
 
 namespace OrcanodeMonitor.Core
 {
     public class FfmpegCoreAnalyzer
     {
+        // We consider anything below this amplitude as silence.
+        const double MaxSilenceAmplitude = 20.0;
+
+        // Microphone audio hum typically falls within the 50 Hz to 60 Hz
+        // range. This hum is often caused by electrical interference from
+        // power lines and other electronic devices.
+        const double MinHumFrequency = 50.0; // Hz
+        const double MaxHumFrequency = 60.0; // Hz
+
+        private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleRate)
+        {
+            int n = data.Length;
+            Complex[] complexData = data.Select(d => new Complex(d, 0)).ToArray();
+            Fourier.Forward(complexData, FourierOptions.Matlab);
+            double[] amplitudes = new double[n / 2];
+            for (int i = 0; i < n / 2; i++)
+            {
+                amplitudes[i] = complexData[i].Magnitude;
+            }
+
+            double max = amplitudes.Max();
+            if (max < MaxSilenceAmplitude)
+            {
+                // File contains mostly silence across all frequencies.
+                return OrcanodeOnlineStatus.Unintelligible;
+            }
+
+            // Look for signal in frequencies other than the audio hum range.
+            double halfOfMax = amplitudes.Max() / 2.0;
+            var majorOtherIndices = new List<int>();
+            for (int i = 0; i < amplitudes.Length; i++)
+            {
+                if (amplitudes[i] > halfOfMax)
+                {
+                    double frequency = (((double)i) * sampleRate) / n;
+                    if (frequency < MinHumFrequency || frequency > MaxHumFrequency)
+                    {
+                        majorOtherIndices.Add(i);
+                    }
+                }
+            }
+
+            if (majorOtherIndices.Count == 0)
+            {
+                // Essentially just silence outside the hum range, no signal.
+                return OrcanodeOnlineStatus.Unintelligible;
+            }
+
+            return OrcanodeOnlineStatus.Online;
+        }
+
         /// <summary>
-        /// Get the Std.Dev. of the most recent audio samples.
+        /// Get the status of the most recent audio stream sample.
         /// </summary>
         /// <param name="args">FFMpeg arguments</param>
-        /// <returns>Std.Dev. of the most recent audio samples</returns>
-        private static async Task<double> AnalyzeAsync(FFMpegArguments args)
+        /// <returns>Status of the most recent audio samples</returns>
+        private static async Task<OrcanodeOnlineStatus> AnalyzeAsync(FFMpegArguments args)
         {
             var outputStream = new MemoryStream(); // Create an output stream (e.g., MemoryStream)
             var pipeSink = new StreamPipeSink(outputStream);
@@ -23,49 +78,41 @@ private static async Task<double> AnalyzeAsync(FFMpegArguments args)
                 .OutputToPipe(pipeSink, options => options
                 .WithAudioCodec("pcm_s16le")
                 .ForceFormat("wav"))
-            .ProcessAsynchronously();
+                .ProcessAsynchronously();
 
             var waveFormat = new WaveFormat(rate: 44100, bits: 16, channels: 1);
             var rawStream = new RawSourceWaveStream(outputStream, waveFormat);
 
-            byte[] buffer = new byte[4096]; // Adjust buffer size as needed
-            int bytesRead;
-
             // Reset the position to the beginning
             rawStream.Seek(0, SeekOrigin.Begin);
 
-            var variance = new WelfordVariance();
+            // Read the audio data into a byte buffer.
+            var byteBuffer = new byte[rawStream.Length];
+            int bytesRead = rawStream.Read(byteBuffer, 0, byteBuffer.Length);
 
-            while ((bytesRead = rawStream.Read(buffer, 0, buffer.Length)) > 0)
+            // Convert byte buffer to float buffer.
+            var floatBuffer = new float[byteBuffer.Length / sizeof(short)];
+            for (int i = 0; i < floatBuffer.Length; i++)
             {
-                for (int i = 0; i < bytesRead; i += 2) // Assuming 16-bit samples
-                {
-                    short sample = BitConverter.ToInt16(buffer, i);
-                    variance.Add(sample);
-                }
+                floatBuffer[i] = BitConverter.ToInt16(byteBuffer, i * sizeof(short)) / 32768f;
             }
-            return variance.StandardDeviation;
+
+            // Perform FFT and analyze frequencies
+            var status = AnalyzeFrequencies(floatBuffer, waveFormat.SampleRate);
+            return status;
         }
 
-        public static async Task<double> AnalyzeFileAsync(string filename)
+        public static async Task<OrcanodeOnlineStatus> AnalyzeFileAsync(string filename)
         {
-#if false
-            var mediaInfo = await FFProbe.AnalyseAsync(filename);
-            if (mediaInfo == null)
-            {
-                return false;
-            }
-#endif
             var args = FFMpegArguments.FromFileInput(filename);
             return await AnalyzeAsync(args);
         }
 
-        public static async Task<double> AnalyzeAudioStreamAsync(Stream stream)
+        public static async Task<OrcanodeOnlineStatus> AnalyzeAudioStreamAsync(Stream stream)
         {
             StreamPipeSource streamPipeSource = new StreamPipeSource(stream);
             var args = FFMpegArguments.FromPipeInput(streamPipeSource);
             return await AnalyzeAsync(args);
         }
     }
-}
-
+}

OrcanodeMonitor/Data/OrcanodeMonitorContext.cs

@@ -38,6 +38,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 .ToContainer("Orcanode")
                 .Property(item => item.ID);
 
+            modelBuilder.Entity<Orcanode>()
+                .ToContainer("Orcanode")
+                .Property(item => item.AudioStreamStatus)
+                .HasDefaultValue(OrcanodeOnlineStatus.Absent);
+
             modelBuilder.Entity<Orcanode>()
                 .ToContainer("Orcanode")
                 .HasPartitionKey(item => item.PartitionValue)

OrcanodeMonitor/Models/Orcanode.cs

@@ -58,7 +58,7 @@ public Orcanode()
             DataplicityName = string.Empty;
             DataplicitySerial = string.Empty;
             OrcaHelloId = string.Empty;
-            PartitionValue = 1;            
+            PartitionValue = 1;
         }
 
         #region persisted
@@ -182,6 +182,11 @@ public Orcanode()
         /// Partition key fixed value.
         /// </summary>
         public int PartitionValue { get; set; }
+
+        /// <summary>
+        /// Audio stream status of most recent sample (defaults to absent).
+        /// </summary>
+        public OrcanodeOnlineStatus? AudioStreamStatus { get; set;  }
 
         #endregion persisted
 
@@ -295,7 +300,14 @@ public OrcanodeOnlineStatus S3StreamStatus
                 {
                     return OrcanodeOnlineStatus.Offline;
                 }
-                return (IsUnintelligible(AudioStandardDeviation)) ? OrcanodeOnlineStatus.Unintelligible : OrcanodeOnlineStatus.Online;
+
+                if (AudioStreamStatus == OrcanodeOnlineStatus.Absent && AudioStandardDeviation != 0.0)
+                {
+                    // Fall back to legacy algorithm.
+                    AudioStreamStatus = (IsUnintelligible(AudioStandardDeviation)) ? OrcanodeOnlineStatus.Unintelligible : OrcanodeOnlineStatus.Online;
+                }
+
+                return AudioStreamStatus ?? OrcanodeOnlineStatus.Absent;
             }
         }
 
@@ -313,6 +325,12 @@ public string OrcasoundOnlineStatusString {
 
         #region methods
 
+        /// <summary>
+        /// Function used for backwards compatibility when reading a database
+        /// entry with no AudioStreamStatus.
+        /// </summary>
+        /// <param name="audioStandardDeviation"></param>
+        /// <returns></returns>
         public static bool IsUnintelligible(double? audioStandardDeviation)
         {
             if (audioStandardDeviation.HasValue && (audioStandardDeviation < MinIntelligibleStreamDeviation))

OrcanodeMonitor/OrcanodeMonitor.csproj

@@ -13,6 +13,7 @@
   <ItemGroup>
     <PackageReference Include="FFMpegCore" Version="5.1.0" />
     <PackageReference Include="FFMpegInstaller.Windows.x64" Version="0.1.10" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Cosmos" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Proxies" Version="8.0.8" />

Test/UnintelligibilityTests.cs

@@ -12,7 +12,7 @@ namespace Test
     [TestClass]
     public class UnintelligibilityTests
     {
-        private async Task TestSampleAsync(string filename, bool expected_result)
+        private async Task TestSampleAsync(string filename, OrcanodeOnlineStatus expected_status)
         {
             // Get the current directory (where the test assembly is located)
             string currentDirectory = Directory.GetCurrentDirectory();
@@ -23,9 +23,8 @@ private async Task TestSampleAsync(string filename, bool expected_result)
             string filePath = Path.Combine(rootDirectory, "Test\\samples", filename);
             try
             {
-                double audioStandardDeviation = await FfmpegCoreAnalyzer.AnalyzeFileAsync(filePath);
-                bool normal = !Orcanode.IsUnintelligible(audioStandardDeviation);
-                Assert.IsTrue(normal ==  expected_result);
+                OrcanodeOnlineStatus status = await FfmpegCoreAnalyzer.AnalyzeFileAsync(filePath);
+                Assert.IsTrue(status ==  expected_status);
             }
             catch (Exception ex)
             {
@@ -39,17 +38,18 @@ private async Task TestSampleAsync(string filename, bool expected_result)
         [TestMethod]
         public async Task TestUnintelligibleSample()
         {
-            await TestSampleAsync("unintelligible\\live1791.ts", false);
-            await TestSampleAsync("unintelligible\\live1815.ts", false);
-            await TestSampleAsync("unintelligible\\live1816.ts", false);
+            await TestSampleAsync("unintelligible\\live1816b.ts", OrcanodeOnlineStatus.Unintelligible);
+            await TestSampleAsync("unintelligible\\live1791.ts", OrcanodeOnlineStatus.Unintelligible);
+            await TestSampleAsync("unintelligible\\live1815.ts", OrcanodeOnlineStatus.Unintelligible);
+            await TestSampleAsync("unintelligible\\live1816.ts", OrcanodeOnlineStatus.Unintelligible);
         }
 
         [TestMethod]
         public async Task TestNormalSample()
         {
-            await TestSampleAsync("normal\\live385.ts", true);
-            await TestSampleAsync("normal\\live839.ts", true);
-            await TestSampleAsync("normal\\live1184.ts", true);
+            await TestSampleAsync("normal\\live385.ts", OrcanodeOnlineStatus.Online);
+            await TestSampleAsync("normal\\live839.ts", OrcanodeOnlineStatus.Online);
+            await TestSampleAsync("normal\\live1184.ts", OrcanodeOnlineStatus.Online);
         }
     }
 }