diff --git a/OrcanodeMonitor/Core/Fetcher.cs b/OrcanodeMonitor/Core/Fetcher.cs index 3878c07..c7f8cf8 100644 --- a/OrcanodeMonitor/Core/Fetcher.cs +++ b/OrcanodeMonitor/Core/Fetcher.cs @@ -21,6 +21,7 @@ using System.Threading.Channels; using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.Logging; +using Microsoft.AspNetCore.Diagnostics; namespace OrcanodeMonitor.Core { @@ -735,42 +736,77 @@ public static long DateTimeToUnixTimeStamp(DateTime dateTime) return unixTime; } - /// - /// Update the timestamps for a given Orcanode by querying files on S3. - /// - /// Database context - /// Orcanode to update - /// - /// - public async static Task UpdateS3DataAsync(OrcanodeMonitorContext context, Orcanode node, ILogger logger) + public class TimestampResult + { + public string UnixTimestampString { get; } + public DateTimeOffset? Offset { get; } + public TimestampResult(string unixTimestampString, DateTimeOffset? offset) + { + UnixTimestampString = unixTimestampString; + Offset = offset; + } + } + + public async static Task GetLatestS3TimestampAsync(Orcanode node, bool updateNode, ILogger logger) { string url = "https://" + node.S3Bucket + ".s3.amazonaws.com/" + node.S3NodeName + "/latest.txt"; using HttpResponseMessage response = await _httpClient.GetAsync(url); if (response.StatusCode == HttpStatusCode.NotFound) { + logger.LogError($"{node.S3NodeName} not found on S3"); + // Absent. - node.LatestRecordedUtc = null; - return; + if (updateNode) + { + node.LatestRecordedUtc = null; + } + return null; } if (response.StatusCode == HttpStatusCode.Forbidden) { + logger.LogError($"{node.S3NodeName} got access denied on S3"); + // Access denied. - node.LatestRecordedUtc = DateTime.MinValue; - return; + if (updateNode) + { + node.LatestRecordedUtc = DateTime.MinValue; + } + return null; } if (!response.IsSuccessStatusCode) { - return; + logger.LogError($"{node.S3NodeName} got status {response.StatusCode} on S3"); + + return null; } string content = await response.Content.ReadAsStringAsync(); string unixTimestampString = content.TrimEnd(); + var result = new TimestampResult(unixTimestampString, response.Content.Headers.LastModified); + return result; + } + + /// + /// Update the timestamps for a given Orcanode by querying files on S3. + /// + /// Database context + /// Orcanode to update + /// + /// + public async static Task UpdateS3DataAsync(OrcanodeMonitorContext context, Orcanode node, ILogger logger) + { + TimestampResult? result = await GetLatestS3TimestampAsync(node, true, logger); + if (result == null) + { + return; + } + string unixTimestampString = result.UnixTimestampString; DateTime? latestRecorded = UnixTimeStampStringToDateTimeUtc(unixTimestampString); if (latestRecorded.HasValue) { node.LatestRecordedUtc = latestRecorded?.ToUniversalTime(); - DateTimeOffset? offset = response.Content.Headers.LastModified; + DateTimeOffset? offset = result.Offset; if (offset.HasValue) { node.LatestUploadedUtc = offset.Value.UtcDateTime; @@ -864,15 +900,7 @@ private static void AddHydrophoneStreamStatusEvent(OrcanodeMonitorContext contex AddOrcanodeEvent(context, node, OrcanodeEventTypes.HydrophoneStream, value); } - /// - /// Update the ManifestUpdated timestamp for a given Orcanode by querying S3. - /// - /// Database context - /// Orcanode to update - /// Value in the latest.txt file - /// Logger - /// - public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext context, Orcanode node, string unixTimestampString, ILogger logger) + public async static Task GetLatestAudioSampleAsync(Orcanode node, string unixTimestampString, bool updateNode, ILogger logger) { OrcanodeOnlineStatus oldStatus = node.S3StreamStatus; @@ -880,18 +908,24 @@ public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext con using HttpResponseMessage response = await _httpClient.GetAsync(url); if (!response.IsSuccessStatusCode) { - return; + return null; } DateTimeOffset? offset = response.Content.Headers.LastModified; if (!offset.HasValue) { - node.LastCheckedUtc = DateTime.UtcNow; - return; + if (updateNode) + { + node.LastCheckedUtc = DateTime.UtcNow; + } + return null; } - node.ManifestUpdatedUtc = offset.Value.UtcDateTime; - node.LastCheckedUtc = DateTime.UtcNow; + if (updateNode) + { + node.ManifestUpdatedUtc = offset.Value.UtcDateTime; + node.LastCheckedUtc = DateTime.UtcNow; + } // Download manifest. Uri baseUri = new Uri(url); @@ -908,14 +942,38 @@ public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext con try { using Stream stream = await _httpClient.GetStreamAsync(newUri); - node.AudioStreamStatus = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream, oldStatus); - node.AudioStandardDeviation = 0.0; - } catch (Exception ex) + FrequencyInfo frequencyInfo = await FfmpegCoreAnalyzer.AnalyzeAudioStreamAsync(stream, oldStatus); + return frequencyInfo; + } + catch (Exception ex) { // We couldn't fetch the stream audio so could not update the // audio standard deviation. Just ignore this for now. logger.LogError(ex, "Exception in UpdateManifestTimestampAsync"); } + return null; + } + + /// + /// Update the ManifestUpdated timestamp for a given Orcanode by querying S3. + /// + /// Database context + /// Orcanode to update + /// Value in the latest.txt file + /// Logger + /// + public async static Task UpdateManifestTimestampAsync(OrcanodeMonitorContext context, Orcanode node, string unixTimestampString, ILogger logger) + { + OrcanodeOnlineStatus oldStatus = node.S3StreamStatus; + + FrequencyInfo? frequencyInfo = await GetLatestAudioSampleAsync(node, unixTimestampString, true, logger); + if (frequencyInfo == null) + { + return; + } + + node.AudioStreamStatus = frequencyInfo.Status; + node.AudioStandardDeviation = 0.0; OrcanodeOnlineStatus newStatus = node.S3StreamStatus; if (newStatus != oldStatus) diff --git a/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs b/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs index 657bdea..5bf41d2 100644 --- a/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs +++ b/OrcanodeMonitor/Core/FfmpegCoreAnalyzer.cs @@ -9,6 +9,12 @@ namespace OrcanodeMonitor.Core { + public class FrequencyInfo + { + public Dictionary FrequencyAmplitudes { get; set; } + public OrcanodeOnlineStatus Status { get; set; } + } + public class FfmpegCoreAnalyzer { // We consider anything above this average amplitude as not silence. @@ -57,36 +63,47 @@ private static double MinSignalRatio private static bool IsHumFrequency(double frequency) => (frequency >= MinHumFrequency && frequency <= MaxHumFrequency); - private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleRate, OrcanodeOnlineStatus oldStatus) + public static Dictionary ComputeFrequencyAmplitudes(float[] data, int sampleRate) { + var result = new Dictionary(); int n = data.Length; Complex[] complexData = data.Select(d => new Complex(d, 0)).ToArray(); Fourier.Forward(complexData, FourierOptions.Matlab); - double[] amplitudes = new double[n / 2]; for (int i = 0; i < n / 2; i++) { - amplitudes[i] = complexData[i].Magnitude; + double amplitude = complexData[i].Magnitude; + double frequency = (((double)i) * sampleRate) / n; + result[frequency] = amplitude; } + return result; + } - double max = amplitudes.Max(); + private static FrequencyInfo AnalyzeFrequencies(float[] data, int sampleRate, OrcanodeOnlineStatus oldStatus) + { + int n = data.Length; + FrequencyInfo frequencyInfo = new FrequencyInfo(); + frequencyInfo.FrequencyAmplitudes = ComputeFrequencyAmplitudes(data, sampleRate); + + double max = frequencyInfo.FrequencyAmplitudes.Values.Max(); if (max < MinNoiseAmplitude) { // File contains mostly silence across all frequencies. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } if ((max <= MaxSilenceAmplitude) && (oldStatus == OrcanodeOnlineStatus.Unintelligible)) { // In between the min and max unintelligibility range, so keep previous status. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } // Find the maximum amplitude outside the audio hum range. double maxNonHumAmplitude = 0; - for (int i = 0; i < amplitudes.Length; i++) - { - double frequency = (((double)i) * sampleRate) / n; - double amplitude = amplitudes[i]; + foreach (var pair in frequencyInfo.FrequencyAmplitudes) { + double frequency = pair.Key; + double amplitude = pair.Value; if (!IsHumFrequency(frequency)) { if (maxNonHumAmplitude < amplitude) @@ -99,11 +116,13 @@ private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleR if (maxNonHumAmplitude / max < MinSignalRatio) { // Essentially just silence outside the hum range, no signal. - return OrcanodeOnlineStatus.Unintelligible; + frequencyInfo.Status = OrcanodeOnlineStatus.Unintelligible; + return frequencyInfo; } // Signal outside the hum range. - return OrcanodeOnlineStatus.Online; + frequencyInfo.Status = OrcanodeOnlineStatus.Online; + return frequencyInfo; } /// @@ -112,7 +131,7 @@ private static OrcanodeOnlineStatus AnalyzeFrequencies(float[] data, int sampleR /// FFMpeg arguments /// Previous online status /// Status of the most recent audio samples - private static async Task AnalyzeAsync(FFMpegArguments args, OrcanodeOnlineStatus oldStatus) + private static async Task AnalyzeAsync(FFMpegArguments args, OrcanodeOnlineStatus oldStatus) { var outputStream = new MemoryStream(); // Create an output stream (e.g., MemoryStream) var pipeSink = new StreamPipeSink(outputStream); @@ -142,18 +161,18 @@ private static async Task AnalyzeAsync(FFMpegArguments arg floatBuffer[i] = BitConverter.ToInt16(byteBuffer, i * sizeof(short)) / 32768f; } - // Perform FFT and analyze frequencies + // Perform FFT and analyze frequencies. var status = AnalyzeFrequencies(floatBuffer, waveFormat.SampleRate, oldStatus); return status; } - public static async Task AnalyzeFileAsync(string filename, OrcanodeOnlineStatus oldStatus) + public static async Task AnalyzeFileAsync(string filename, OrcanodeOnlineStatus oldStatus) { var args = FFMpegArguments.FromFileInput(filename); return await AnalyzeAsync(args, oldStatus); } - public static async Task AnalyzeAudioStreamAsync(Stream stream, OrcanodeOnlineStatus oldStatus) + public static async Task AnalyzeAudioStreamAsync(Stream stream, OrcanodeOnlineStatus oldStatus) { StreamPipeSource streamPipeSource = new StreamPipeSource(stream); var args = FFMpegArguments.FromPipeInput(streamPipeSource); diff --git a/OrcanodeMonitor/Pages/NodeEvents.cshtml b/OrcanodeMonitor/Pages/NodeEvents.cshtml index ac4c22d..1f2f927 100644 --- a/OrcanodeMonitor/Pages/NodeEvents.cshtml +++ b/OrcanodeMonitor/Pages/NodeEvents.cshtml @@ -5,56 +5,46 @@ }
-

Node Events

-
- @Html.AntiForgeryToken() - - -
- - -
-
-

-

- @Html.AntiForgeryToken() - - -
- - - - -
-
-

- Uptime percentage: @Model.UptimePercentage% -

- +

@Model.NodeName

+ + + +

+ +
+ Uptime percentage: + + + + + @Model.GetUptimePercentage("all", "pastMonth")% + + + +

+
+ + +
+ Filter by Time Range: + + +

+
+ + +
+ Filter by Type: + + + + +

+
+ +
@@ -70,11 +60,81 @@ } @foreach (Models.OrcanodeEvent item in Model.RecentEvents) { - + }
Timestamp (Pacific)
@item.DateTimeLocal.ToString("g") @item.Description
+ +
diff --git a/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs b/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs index 2ca86af..0e0aa2d 100644 --- a/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs +++ b/OrcanodeMonitor/Pages/NodeEvents.cshtml.cs @@ -2,10 +2,10 @@ // SPDX-License-Identifier: MIT using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.RazorPages; -using Microsoft.IdentityModel.Tokens; using OrcanodeMonitor.Core; using OrcanodeMonitor.Data; using OrcanodeMonitor.Models; +using static OrcanodeMonitor.Core.Fetcher; namespace OrcanodeMonitor.Pages { @@ -13,8 +13,9 @@ public class NodeEventsModel : PageModel { private readonly OrcanodeMonitorContext _databaseContext; private readonly ILogger _logger; - private string _nodeId; - public string Id => _nodeId; + private Orcanode? _node = null; + public string Id => _node?.ID ?? string.Empty; + public string NodeName => _node?.DisplayName ?? "Unknown"; [BindProperty] public string TimePeriod { get; set; } = "week"; // Default to 'week' @@ -24,57 +25,59 @@ public class NodeEventsModel : PageModel private DateTime SinceTime => (TimePeriod == "week") ? DateTime.UtcNow.AddDays(-7) : DateTime.UtcNow.AddMonths(-1); private List _events; public List RecentEvents => _events; - public int UptimePercentage => Orcanode.GetUptimePercentage(_nodeId, _events, SinceTime, (EventType == OrcanodeEventTypes.All) ? OrcanodeEventTypes.HydrophoneStream : EventType); + public int GetUptimePercentage(string type, string timeRange) + { + DateTime sinceTime = (timeRange == "pastWeek") ? DateTime.UtcNow.AddDays(-7) : DateTime.UtcNow.AddMonths(-1); + string eventType = type switch + { + "hydrophoneStream" => OrcanodeEventTypes.HydrophoneStream, + "dataplicityConnection" => OrcanodeEventTypes.DataplicityConnection, + "mezmoLogging" => OrcanodeEventTypes.MezmoLogging, + _ => OrcanodeEventTypes.HydrophoneStream + }; + return Orcanode.GetUptimePercentage(Id, _events, sinceTime, eventType); + } public NodeEventsModel(OrcanodeMonitorContext context, ILogger logger) { _databaseContext = context; _logger = logger; - _nodeId = string.Empty; _events = new List(); } private void FetchEvents(ILogger logger) { - _events = Fetcher.GetRecentEventsForNode(_databaseContext, _nodeId, SinceTime, logger).Where(e => e.Type == EventType || EventType == OrcanodeEventTypes.All).ToList() ?? new List(); + _events = Fetcher.GetRecentEventsForNode(_databaseContext, Id, SinceTime, logger) + .Where(e => e.Type == EventType || EventType == OrcanodeEventTypes.All) + .ToList() ?? new List(); } - public void OnGet(string id) + public async Task OnGetAsync(string id) { - _nodeId = id; + _node = _databaseContext.Orcanodes.Where(n => n.ID == id).First(); FetchEvents(_logger); } - public IActionResult OnPost(string timePeriod, string eventType, string id) + public string GetTypeClass(OrcanodeEvent item) => item.Type switch { - if (string.IsNullOrEmpty(id)) - { - _logger.LogError("Node ID cannot be empty"); - return BadRequest("Invalid node ID"); - } - if (timePeriod.IsNullOrEmpty()) - { - timePeriod = TimePeriod; - } - if (eventType.IsNullOrEmpty()) - { - eventType = EventType; - } - if (timePeriod != "week" && timePeriod != "month") - { - _logger.LogWarning($"Invalid time range selected: {timePeriod}"); - return BadRequest("Invalid time range"); - } - if (eventType != OrcanodeEventTypes.All && eventType != OrcanodeEventTypes.HydrophoneStream && eventType != OrcanodeEventTypes.MezmoLogging && eventType != OrcanodeEventTypes.DataplicityConnection) - { - _logger.LogWarning($"Invalid event type selected: {eventType}"); - return BadRequest("Invalid event type"); - } - TimePeriod = timePeriod; - EventType = eventType; - _nodeId = id; - FetchEvents(_logger); - return Page(); + OrcanodeEventTypes.HydrophoneStream => "hydrophoneStream", + OrcanodeEventTypes.DataplicityConnection => "dataplicityConnection", + OrcanodeEventTypes.MezmoLogging => "mezmoLogging", + OrcanodeEventTypes.AgentUpgradeStatus => "agentUpgradeStatus", + OrcanodeEventTypes.SDCardSize => "sdCardSize", + _ => string.Empty + }; + + public string GetTimeRangeClass(OrcanodeEvent item) + { + DateTime OneWeekAgo = DateTime.UtcNow.AddDays(-7); + return (item.DateTimeUtc > OneWeekAgo) ? "pastWeek" : string.Empty; + } + + public string GetEventClasses(OrcanodeEvent item) + { + string classes = GetTypeClass(item) + " " + GetTimeRangeClass(item); + return classes; } } } diff --git a/OrcanodeMonitor/Pages/SpectralDensity.cshtml b/OrcanodeMonitor/Pages/SpectralDensity.cshtml new file mode 100644 index 0000000..7dfcad1 --- /dev/null +++ b/OrcanodeMonitor/Pages/SpectralDensity.cshtml @@ -0,0 +1,48 @@ +@page "{id}" +@model OrcanodeMonitor.Pages.SpectralDensityModel +@{ + ViewData["Title"] = "Spectral Density"; +} + +
+

Spectral Density

+ + + + + + + +
diff --git a/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs b/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs new file mode 100644 index 0000000..addf522 --- /dev/null +++ b/OrcanodeMonitor/Pages/SpectralDensity.cshtml.cs @@ -0,0 +1,91 @@ +// Copyright (c) Orcanode Monitor contributors +// SPDX-License-Identifier: MIT +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using OrcanodeMonitor.Core; +using OrcanodeMonitor.Data; +using OrcanodeMonitor.Models; +using static OrcanodeMonitor.Core.Fetcher; + +namespace OrcanodeMonitor.Pages +{ + /// + /// Razor Page model for spectral density visualization. + /// Handles retrieval and processing of frequency data for display. + /// + public class SpectralDensityModel : PageModel + { + private readonly OrcanodeMonitorContext _databaseContext; + private readonly ILogger _logger; + private string _nodeId; + public string Id => _nodeId; + private List _labels; + private List _data; + public List Labels => _labels; + public List Data => _data; + + public SpectralDensityModel(OrcanodeMonitorContext context, ILogger logger) + { + _databaseContext = context; + _logger = logger; + _nodeId = string.Empty; + } + + private async Task UpdateFrequencyDataAsync() + { + _labels = new List { }; + _data = new List { }; + Orcanode? node = _databaseContext.Orcanodes.Where(n => n.ID == _nodeId).FirstOrDefault(); + if (node == null) + { + _logger.LogWarning("Node not found with ID: {NodeId}", _nodeId); + return; + } + TimestampResult? result = await GetLatestS3TimestampAsync(node, false, _logger); + if (result != null) + { + FrequencyInfo? frequencyInfo = await Fetcher.GetLatestAudioSampleAsync(node, result.UnixTimestampString, false, _logger); + if (frequencyInfo != null) + { + const int MaxFrequency = 23000; + const int PointCount = 1000; + + // Compute the logarithmic base needed to get PointCount points. + double b = Math.Pow(MaxFrequency, 1.0 / PointCount); + double logb = Math.Log(b); + + double maxAmplitude = frequencyInfo.FrequencyAmplitudes.Values.Max(); + var sums = new double[PointCount]; + var count = new int[PointCount]; + + foreach (var pair in frequencyInfo.FrequencyAmplitudes) + { + double frequency = pair.Key; + double amplitude = pair.Value; + int bucket = (frequency < 1) ? 0 : (int)(Math.Log(frequency) / logb); + count[bucket]++; + sums[bucket] += amplitude; + } + + // Fill in graph points. + for (int i = 0; i < PointCount; i++) + { + if (count[i] > 0) + { + int frequency = (int)Math.Pow(b, i); + int amplitude = (int)(sums[i] / count[i]); + _labels.Add(frequency.ToString()); + _data.Add(amplitude); + } + } + } + } + } + + public async Task OnGetAsync(string id) + { + _nodeId = id; + await UpdateFrequencyDataAsync(); + } + } +} diff --git a/Test/UnintelligibilityTests.cs b/Test/UnintelligibilityTests.cs index 855c9a0..c4efe67 100644 --- a/Test/UnintelligibilityTests.cs +++ b/Test/UnintelligibilityTests.cs @@ -24,7 +24,8 @@ private async Task TestSampleAsync(string filename, OrcanodeOnlineStatus expecte try { OrcanodeOnlineStatus previousStatus = oldStatus ?? expected_status; - OrcanodeOnlineStatus status = await FfmpegCoreAnalyzer.AnalyzeFileAsync(filePath, previousStatus); + FrequencyInfo frequencyInfo = await FfmpegCoreAnalyzer.AnalyzeFileAsync(filePath, previousStatus); + OrcanodeOnlineStatus status = frequencyInfo.Status; Assert.IsTrue(status == expected_status); } catch (Exception ex)