diff --git a/OrcanodeMonitor/Core/Fetcher.cs b/OrcanodeMonitor/Core/Fetcher.cs index 3878c07..5053f96 100644 --- a/OrcanodeMonitor/Core/Fetcher.cs +++ b/OrcanodeMonitor/Core/Fetcher.cs @@ -21,6 +21,7 @@ using System.Threading.Channels; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Logging; +using Microsoft.AspNetCore.Diagnostics; namespace OrcanodeMonitor.Core { @@ -735,42 +736,68 @@ public static long DateTimeToUnixTimeStamp(DateTime dateTime) return unixTime; } - /// - /// Update the timestamps for a given Orcanode by querying files on S3. - /// - /// Database context - /// Orcanode to update - /// - /// - public async static Task UpdateS3DataAsync(OrcanodeMonitorContext context, Orcanode node, ILogger logger) + public class TimestampResult + { + public string UnixTimestampString; + public DateTimeOffset? Offset; + } + + public async static Task GetLatestS3TimestampAsync(Orcanode node, bool updateNode, ILogger logger) { string url = "https://" + node.S3Bucket + ".s3.amazonaws.com/" + node.S3NodeName + "/latest.txt"; using HttpResponseMessage response = await _httpClient.GetAsync(url); if (response.StatusCode == HttpStatusCode.NotFound) { // Absent. - node.LatestRecordedUtc = null; - return; + if (updateNode) + { + node.LatestRecordedUtc = null; + } + return null; } if (response.StatusCode == HttpStatusCode.Forbidden) { // Access denied. - node.LatestRecordedUtc = DateTime.MinValue; - return; + if (updateNode) + { + node.LatestRecordedUtc = DateTime.MinValue; + } + return null; } if (!response.IsSuccessStatusCode) { - return; + return null; } string content = await response.Content.ReadAsStringAsync(); string unixTimestampString = content.TrimEnd(); + var result = new TimestampResult(); + result.UnixTimestampString = unixTimestampString; + result.Offset = response.Content.Headers.LastModified; + return result; + } + + /// + /// Update the timestamps for a given Orcanode by querying files on S3. + /// + /// Database context + /// Orcanode to update + /// + /// + public async static Task UpdateS3DataAsync(OrcanodeMonitorContext context, Orcanode node, ILogger logger) + { + TimestampResult? result = await GetLatestS3TimestampAsync(node, true, logger); + if (result == null) + { + return; + } + string unixTimestampString = result.UnixTimestampString; DateTime? latestRecorded = UnixTimeStampStringToDateTimeUtc(unixTimestampString); if (latestRecorded.HasValue) { node.LatestRecordedUtc = latestRecorded?.ToUniversalTime(); - DateTimeOffset? offset = response.Content.Headers.LastModified; + DateTimeOffset? offset = result.Offset; if (offset.HasValue) { node.LatestUploadedUtc = offset.Value.UtcDateTime; @@ -864,15 +891,7 @@ private static void AddHydrophoneStreamStatusEvent(OrcanodeMonitorContext contex AddOrcanodeEvent(context, node, OrcanodeEventTypes.HydrophoneStream, value); } - /// - /// Update the ManifestUpdated timestamp for a given Orcanode by querying S3. - /// - /// Database context - /// Orcanode to update - /// Value in the latest.txt file - /// Logger - /// - public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext context, Orcanode node, string unixTimestampString, ILogger logger) + public async static Task GetLatestAudioSampleAsync(Orcanode node, string unixTimestampString, bool updateNode, ILogger logger) { OrcanodeOnlineStatus oldStatus = node.S3StreamStatus; @@ -880,18 +899,24 @@ public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext con using HttpResponseMessage response = await _httpClient.GetAsync(url); if (!response.IsSuccessStatusCode) { - return; + return null; } DateTimeOffset? offset = response.Content.Headers.LastModified; if (!offset.HasValue) { - node.LastCheckedUtc = DateTime.UtcNow; - return; + if (updateNode) + { + node.LastCheckedUtc = DateTime.UtcNow; + } + return null; } - node.ManifestUpdatedUtc = offset.Value.UtcDateTime; - node.LastCheckedUtc = DateTime.UtcNow; + if (updateNode) + { + node.ManifestUpdatedUtc = offset.Value.UtcDateTime; + node.LastCheckedUtc = DateTime.UtcNow; + } // Download manifest. Uri baseUri = new Uri(url); @@ -908,14 +933,38 @@ public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext con try { using Stream stream = await _httpClient.GetStreamAsync(newUri); - node.AudioStreamStatus = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream, oldStatus); - node.AudioStandardDeviation = 0.0; - } catch (Exception ex) + FrequencyInfo frequencyInfo = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream, oldStatus); + return frequencyInfo; + } + catch (Exception ex) { // We couldn't fetch the stream audio so could not update the // audio standard deviation. Just ignore this for now. logger.LogError(ex, "Exception in UpdateManifestTimestampAsync"); } + return null; + } + + /// + /// Update the ManifestUpdated timestamp for a given Orcanode by querying S3. + /// + /// Database context + /// Orcanode to update + /// Value in the latest.txt file + /// Logger + /// + public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext context, Orcanode node, string unixTimestampString, ILogger logger) + { + OrcanodeOnlineStatus oldStatus = node.S3StreamStatus; + + FrequencyInfo? frequencyInfo = await GetLatestAudioSampleAsync(node, unixTimestampString, true, logger); + if (frequencyInfo == null) + { + return; + } + + node.AudioStreamStatus = frequencyInfo.Status; + node.AudioStandardDeviation = 0.0; OrcanodeOnlineStatus newStatus = node.S3StreamStatus; if (newStatus != oldStatus) diff --git a/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs b/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs index 657bdea..5bf41d2 100644 --- a/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs +++ b/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs @@ -9,6 +9,12 @@ namespace OrcanodeMonitor.Core { + public class FrequencyInfo + { + public Dictionary FrequencyAmplitudes { get; set; } + public OrcanodeOnlineStatus Status { get; set; } + } + public class FfmpegCoreAnalyzer { // We consider anything above this average amplitude as not silence. @@ -57,36 +63,47 @@ private static double MinSignalRatio private static bool IsHumFrequency(double frequency) => (frequency >= MinHumFrequency && frequency <= MaxHumFrequency); - private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleRate, OrcanodeOnlineStatus oldStatus) + public static Dictionary ComputeFrequencyAmplitudes(float[] data, int sampleRate) { + var result = new Dictionary(); int n = data.Length; Complex[] complexData = data.Select(d => new Complex(d, 0)).ToArray(); Fourier.Forward(complexData, FourierOptions.Matlab); - double[] amplitudes = new double[n / 2]; for (int i = 0; i < n / 2; i++) { - amplitudes[i] = complexData[i].Magnitude; + double amplitude = complexData[i].Magnitude; + double frequency = (((double)i) * sampleRate) / n; + result[frequency] = amplitude; } + return result; + } - double max = amplitudes.Max(); + private static FrequencyInfo AnalyzeFrequencies(float[] data, int sampleRate, OrcanodeOnlineStatus oldStatus) + { + int n = data.Length; + FrequencyInfo frequencyInfo = new FrequencyInfo(); + frequencyInfo.FrequencyAmplitudes = ComputeFrequencyAmplitudes(data, sampleRate); + + double max = frequencyInfo.FrequencyAmplitudes.Values.Max(); if (max < MinNoiseAmplitude) { // File contains mostly silence across all frequencies. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } if ((max <= MaxSilenceAmplitude) && (oldStatus == OrcanodeOnlineStatus.Unintelligible)) { // In between the min and max unintelligibility range, so keep previous status. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } // Find the maximum amplitude outside the audio hum range. double maxNonHumAmplitude = 0; - for (int i = 0; i < amplitudes.Length; i++) - { - double frequency = (((double)i) * sampleRate) / n; - double amplitude = amplitudes[i]; + foreach (var pair in frequencyInfo.FrequencyAmplitudes) { + double frequency = pair.Key; + double amplitude = pair.Value; if (!IsHumFrequency(frequency)) { if (maxNonHumAmplitude < amplitude) @@ -99,11 +116,13 @@ private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleR if (maxNonHumAmplitude / max < MinSignalRatio) { // Essentially just silence outside the hum range, no signal. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } // Signal outside the hum range. - return OrcanodeOnlineStatus.Online; + frequencyInfo.Status = OrcanodeOnlineStatus.Online; + return frequencyInfo; } /// @@ -112,7 +131,7 @@ private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleR /// FFMpeg arguments /// Previous online status /// Status of the most recent audio samples - private static async Task AnalyzeAsync(FFMpegArguments args, OrcanodeOnlineStatus oldStatus) + private static async Task AnalyzeAsync(FFMpegArguments args, OrcanodeOnlineStatus oldStatus) { var outputStream = new MemoryStream(); // Create an output stream (e.g., MemoryStream) var pipeSink = new StreamPipeSink(outputStream); @@ -142,18 +161,18 @@ private static async Task AnalyzeAsync(FFMpegArguments arg floatBuffer[i] = BitConverter.ToInt16(byteBuffer, i * sizeof(short)) / 32768f; } - // Perform FFT and analyze frequencies + // Perform FFT and analyze frequencies. var status = AnalyzeFrequencies(floatBuffer, waveFormat.SampleRate, oldStatus); return status; } - public static async Task AnalyzeFileAsync(string filename, OrcanodeOnlineStatus oldStatus) + public static async Task AnalyzeFileAsync(string filename, OrcanodeOnlineStatus oldStatus) { var args = FFMpegArguments.FromFileInput(filename); return await AnalyzeAsync(args, oldStatus); } - public static async Task AnalyzeAudioStreamAsync(Stream stream, OrcanodeOnlineStatus oldStatus) + public static async Task AnalyzeAudioStreamAsync(Stream stream, OrcanodeOnlineStatus oldStatus) { StreamPipeSource streamPipeSource = new StreamPipeSource(stream); var args = FFMpegArguments.FromPipeInput(streamPipeSource); diff --git a/OrcanodeMonitor/Pages/NodeEvents.cshtml b/OrcanodeMonitor/Pages/NodeEvents.cshtml index ac4c22d..a3e25cc 100644 --- a/OrcanodeMonitor/Pages/NodeEvents.cshtml +++ b/OrcanodeMonitor/Pages/NodeEvents.cshtml @@ -5,7 +5,14 @@ }
-

Node Events

+

@Model.NodeName

+ + + +

+
@Html.AntiForgeryToken() diff --git a/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs b/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs index 2ca86af..f524048 100644 --- a/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs +++ b/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs @@ -1,11 +1,14 @@ // Copyright (c) Orcanode Monitor contributors // SPDX-License-Identifier: MIT +using MathNet.Numerics.Statistics; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.RazorPages; using Microsoft.IdentityModel.Tokens; using OrcanodeMonitor.Core; using OrcanodeMonitor.Data; using OrcanodeMonitor.Models; +using System.Xml.Linq; +using static OrcanodeMonitor.Core.Fetcher; namespace OrcanodeMonitor.Pages { @@ -13,8 +16,9 @@ public class NodeEventsModel : PageModel { private readonly OrcanodeMonitorContext _databaseContext; private readonly ILogger _logger; - private string _nodeId; - public string Id => _nodeId; + private Orcanode? _node = null; + public string Id => _node?.ID ?? string.Empty; + public string NodeName => _node?.DisplayName ?? "Unknown"; [BindProperty] public string TimePeriod { get; set; } = "week"; // Default to 'week' @@ -24,24 +28,23 @@ public class NodeEventsModel : PageModel private DateTime SinceTime => (TimePeriod == "week") ? DateTime.UtcNow.AddDays(-7) : DateTime.UtcNow.AddMonths(-1); private List _events; public List RecentEvents => _events; - public int UptimePercentage => Orcanode.GetUptimePercentage(_nodeId, _events, SinceTime, (EventType == OrcanodeEventTypes.All) ? OrcanodeEventTypes.HydrophoneStream : EventType); + public int UptimePercentage => Orcanode.GetUptimePercentage(Id, _events, SinceTime, (EventType == OrcanodeEventTypes.All) ? OrcanodeEventTypes.HydrophoneStream : EventType); public NodeEventsModel(OrcanodeMonitorContext context, ILogger logger) { _databaseContext = context; _logger = logger; - _nodeId = string.Empty; _events = new List(); } private void FetchEvents(ILogger logger) { - _events = Fetcher.GetRecentEventsForNode(_databaseContext, _nodeId, SinceTime, logger).Where(e => e.Type == EventType || EventType == OrcanodeEventTypes.All).ToList() ?? new List(); + _events = Fetcher.GetRecentEventsForNode(_databaseContext, Id, SinceTime, logger).Where(e => e.Type == EventType || EventType == OrcanodeEventTypes.All).ToList() ?? new List(); } - public void OnGet(string id) + public async Task OnGetAsync(string id) { - _nodeId = id; + _node = _databaseContext.Orcanodes.Where(n => n.ID == id).First(); FetchEvents(_logger); } @@ -72,7 +75,7 @@ public IActionResult OnPost(string timePeriod, string eventType, string id) } TimePeriod = timePeriod; EventType = eventType; - _nodeId = id; + _node = _databaseContext.Orcanodes.Where(n => n.ID == id).First(); FetchEvents(_logger); return Page(); } diff --git a/OrcanodeMonitor/Pages/SpectralDensity.cshtml b/OrcanodeMonitor/Pages/SpectralDensity.cshtml new file mode 100644 index 0000000..7dfcad1 --- /dev/null +++ b/OrcanodeMonitor/Pages/SpectralDensity.cshtml @@ -0,0 +1,48 @@ +@page "{id}" +@model OrcanodeMonitor.Pages.SpectralDensityModel +@{ + ViewData["Title"] = "Spectral Density"; +} + +
+

Spectral Density

+ + + + + + + +
diff --git a/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs b/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs new file mode 100644 index 0000000..8b0c858 --- /dev/null +++ b/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs @@ -0,0 +1,82 @@ +// Copyright (c) Orcanode Monitor contributors +// SPDX-License-Identifier: MIT +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using OrcanodeMonitor.Core; +using OrcanodeMonitor.Data; +using OrcanodeMonitor.Models; +using static OrcanodeMonitor.Core.Fetcher; + +namespace OrcanodeMonitor.Pages +{ + public class SpectralDensityModel : PageModel + { + private readonly OrcanodeMonitorContext _databaseContext; + private readonly ILogger _logger; + private string _nodeId; + public string Id => _nodeId; + private List _labels; + private List _data; + public List Labels => _labels; + public List Data => _data; + + public SpectralDensityModel(OrcanodeMonitorContext context, ILogger logger) + { + _databaseContext = context; + _logger = logger; + _nodeId = string.Empty; + } + + private async Task UpdateFrequencyDataAsync() + { + _labels = new List { }; + _data = new List { }; + Orcanode node = _databaseContext.Orcanodes.Where(n => n.ID == _nodeId).First(); + TimestampResult? result = await GetLatestS3TimestampAsync(node, false, _logger); + if (result != null) + { + FrequencyInfo? frequencyInfo = await Fetcher.GetLatestAudioSampleAsync(node, result.UnixTimestampString, false, _logger); + if (frequencyInfo != null) + { + const int MaxFrequency = 23000; + const int PointCount = 1000; + + // Compute the logarithmic base needed to get PointCount points. + double b = Math.Pow(MaxFrequency, 1.0 / PointCount); + double logb = Math.Log(b); + + double maxAmplitude = frequencyInfo.FrequencyAmplitudes.Values.Max(); + var sums = new double[PointCount]; + var count = new int[PointCount]; + + foreach (var pair in frequencyInfo.FrequencyAmplitudes) + { + double frequency = pair.Key; + double amplitude = pair.Value; + int bucket = (frequency < 1) ? 0 : (int)(Math.Log(frequency) / logb); + count[bucket]++; + sums[bucket] += amplitude; + } + + // Fill in graph points. + for (int i = 0; i < PointCount; i++) + { + if (count[i] > 0) + { + int frequency = (int)Math.Pow(b, i); + int amplitude = (int)(sums[i] / count[i]); + _labels.Add(frequency.ToString()); + _data.Add(amplitude); + } + } + } + } + } + + public async Task OnGetAsync(string id) + { + _nodeId = id; + await UpdateFrequencyDataAsync(); + } + } +}