Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add OrcaHello column to the nodes table #122

Merged
merged 1 commit into from
Sep 18, 2024
Merged

Add OrcaHello column to the nodes table #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 77 additions & 62 deletions OrcanodeMonitor/Core/Fetcher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class Fetcher
private static HttpClient _httpClient = new HttpClient();
private static string _orcasoundFeedsUrl = "https://live.orcasound.net/api/json/feeds";
private static string _dataplicityDevicesUrl = "https://apps.dataplicity.com/devices/";
private static string _orcaHelloHydrophonesUrl = "https://aifororcasdetections2.azurewebsites.net/api/hydrophones";
private static DateTime _unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
private static string _iftttServiceKey = Environment.GetEnvironmentVariable("IFTTT_SERVICE_KEY") ?? "<unknown>";
private static string _defaultS3Bucket = "streaming-orcasound-net";
Expand All @@ -55,13 +56,12 @@ private static bool NameMatch(string name, string dataplicityName)
/// <summary>
/// Find a node using the serial number value at Dataplicity.
/// </summary>
/// <param name="nodeList">Database table to look in and potentially update</param>
/// <param name="nodes">Database table to look in and potentially update</param>
/// <param name="serial">Dataplicity serial number to look for</param>
/// <param name="connectionStatus">Dataplicity connection status</param>
/// <returns></returns>
private static Orcanode? FindOrcanodeByDataplicitySerial(DbSet<Orcanode> nodeList, string serial, out OrcanodeOnlineStatus connectionStatus)
private static Orcanode? FindOrcanodeByDataplicitySerial(List<Orcanode> nodes, string serial, out OrcanodeOnlineStatus connectionStatus)
{
List<Orcanode> nodes = nodeList.ToList();
foreach (Orcanode node in nodes)
{
if (node.DataplicitySerial == serial)
Expand Down Expand Up @@ -96,7 +96,7 @@ private static Orcanode CreateOrcanode(DbSet<Orcanode> nodeList)
/// <returns></returns>
private static Orcanode FindOrCreateOrcanodeByDataplicitySerial(DbSet<Orcanode> nodeList, string serial, out OrcanodeOnlineStatus connectionStatus)
{
Orcanode? node = FindOrcanodeByDataplicitySerial(nodeList, serial, out connectionStatus);
Orcanode? node = FindOrcanodeByDataplicitySerial(nodeList.ToList(), serial, out connectionStatus);
if (node != null)
{
return node;
Expand All @@ -108,9 +108,8 @@ private static Orcanode FindOrCreateOrcanodeByDataplicitySerial(DbSet<Orcanode>
return newNode;
}

private static Orcanode? FindOrcanodeByOrcasoundFeedId(DbSet<Orcanode> nodeList, string feedId)
private static Orcanode? FindOrcanodeByOrcasoundFeedId(List<Orcanode> nodes, string feedId)
{
List<Orcanode> nodes = nodeList.ToList();
foreach (Orcanode node in nodes)
{
if (node.OrcasoundFeedId == feedId)
Expand All @@ -125,12 +124,11 @@ private static Orcanode FindOrCreateOrcanodeByDataplicitySerial(DbSet<Orcanode>
/// <summary>
/// Look for an Orcanode by Orcasound name in a list.
/// </summary>
/// <param name="nodeList">Orcanode list to look in</param>
/// <param name="nodes">Orcanode list to look in</param>
/// <param name="orcasoundName">Name to look for</param>
/// <returns>Node found</returns>
private static Orcanode? FindOrcanodeByOrcasoundName(DbSet<Orcanode> nodeList, string orcasoundName)
private static Orcanode? FindOrcanodeByOrcasoundName(List<Orcanode> nodes, string orcasoundName)
{
List<Orcanode> nodes = nodeList.ToList();
foreach (Orcanode node in nodes)
{
if (node.OrcasoundName == orcasoundName)
Expand Down Expand Up @@ -176,90 +174,103 @@ private static Orcanode FindOrCreateOrcanodeByOrcasoundName(DbSet<Orcanode> node
return newNode;
}

#if ORCAHELLO
/// <summary>
/// Update the list of Orcanodes using data from OrcaHello.
/// OrcaHello does not currently allow enumerating nodes.
/// </summary>
/// <param name="context">Database context to update</param>
/// <returns></returns>
public async static Task UpdateOrcaHelloDataAsync(OrcanodeMonitorContext context)
{
List<Orcanode> nodes = await context.Orcanodes.ToListAsync();
foreach (Orcanode node in nodes)
{
await UpdateOrcaHelloDataAsync(context, node);
}
}

public async static Task UpdateOrcaHelloDataAsync(OrcanodeMonitorContext context, Orcanode node)
{
string? name = HttpUtility.UrlEncode(node.OrcaHelloName);
if (name == null)
{
return;
}
string url = "https://aifororcasdetections.azurewebsites.net/api/detections?Page=1&SortBy=timestamp&SortOrder=desc&Timeframe=all&Location=" + name + "&RecordsPerPage=1";
string json = "";
try
{
json = await _httpClient.GetStringAsync(url);
if (json == "")
string json = await _httpClient.GetStringAsync(_orcaHelloHydrophonesUrl);
if (json.IsNullOrEmpty())
{
return;
}
dynamic dataArray = JsonSerializer.Deserialize<JsonElement>(json);
if (dataArray.ValueKind != JsonValueKind.Array)
dynamic response = JsonSerializer.Deserialize<ExpandoObject>(json);
if (response == null)
{
return;
}
JsonElement hydrophoneArray = response.hydrophones;
if (hydrophoneArray.ValueKind != JsonValueKind.Array)
{
return;
}
foreach (JsonElement detection in dataArray.EnumerateArray())

// Get a snapshot to use during the loop to avoid multiple queries.
var foundList = context.Orcanodes.ToList();

// Create a list to track what nodes are no longer returned.
var unfoundList = foundList.ToList();

foreach (JsonElement hydrophone in hydrophoneArray.EnumerateArray())
{
if (!detection.TryGetProperty("timestamp", out var timestampElement))
// "id" holds the OrcaHello id which is also the S3NodeName.
if (!hydrophone.TryGetProperty("id", out var hydrophoneId))
{
return;
continue;
}

// "name" holds the display name.
if (!hydrophone.TryGetProperty("name", out var name))
{
continue;
}

// Remove the returned node from the unfound list.
Orcanode? oldListNode = unfoundList.Find(a => a.OrcaHelloId == hydrophoneId.ToString());
if (oldListNode == null)
{
oldListNode = unfoundList.Find(a => a.S3NodeName == hydrophoneId.ToString());
}
if (!DateTime.TryParseExact(timestampElement.ToString(), "yyyy-MM-ddTHH:mm:ss.ffffffZ", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out DateTime timestamp))
if (oldListNode == null)
{
return;
oldListNode = unfoundList.Find(a => a.OrcasoundName == name.ToString());
}
if (timestamp <= node.LastOrcaHelloDetectionTimestamp)
if (oldListNode != null)
{
// No new detections.
return;
unfoundList.Remove(oldListNode);
}

// Parse other properties.
if (!detection.TryGetProperty("confidence", out var confidenceElement))
// TODO: we should have a unique id, independent of S3.
Orcanode? node = FindOrcanodeByOrcasoundName(foundList, name.ToString());
if (node == null)
{
return;
node = CreateOrcanode(context.Orcanodes);
node.OrcasoundName = name.ToString();
node.S3NodeName = hydrophoneId.ToString();
}
if (!confidenceElement.TryGetDouble(out double confidence))

node.OrcaHelloId = hydrophoneId.ToString();
}

// Mark any remaining unfound nodes as absent.
foreach (var unfoundNode in unfoundList)
{
Orcanode? oldNode = null;
if (!unfoundNode.OrcasoundName.IsNullOrEmpty())
{
return;
oldNode = FindOrcanodeByOrcasoundName(foundList, unfoundNode.OrcasoundName);
}
if (!detection.TryGetProperty("comments", out var comments))
else if (!unfoundNode.DataplicitySerial.IsNullOrEmpty())
{
return;
oldNode = FindOrcanodeByDataplicitySerial(foundList, unfoundNode.DataplicitySerial, out OrcanodeOnlineStatus connectionStatus);
}
if (!detection.TryGetProperty("found", out var found))
if (oldNode != null)
{
return;
oldNode.OrcaHelloId = String.Empty;
}

node.LastOrcaHelloDetectionTimestamp = timestamp;
node.LastOrcaHelloDetectionFound = found.ToString() == "yes";
node.LastOrcaHelloDetectionComments = comments.ToString();
node.LastOrcaHelloDetectionConfidence = (int)(confidence + 0.5);
}

await context.SaveChangesAsync();
}
catch (Exception ex)
{
string msg = ex.ToString();
}
}
#endif

/// <summary>
/// Update Orcanode state by querying dataplicity.com.
Expand Down Expand Up @@ -289,8 +300,10 @@ public async static Task UpdateDataplicityDataAsync(OrcanodeMonitorContext conte
jsonArray = await response.Content.ReadAsStringAsync();
}

var foundList = context.Orcanodes.ToList();

// Create a list to track what nodes are no longer returned.
var unfoundList = context.Orcanodes.ToList();
var unfoundList = foundList.ToList();

dynamic deviceArray = JsonSerializer.Deserialize<JsonElement>(jsonArray);
if (deviceArray.ValueKind != JsonValueKind.Array)
Expand Down Expand Up @@ -388,7 +401,7 @@ public async static Task UpdateDataplicityDataAsync(OrcanodeMonitorContext conte
// Mark any remaining unfound nodes as absent.
foreach (var unfoundNode in unfoundList)
{
var oldNode = FindOrcanodeByDataplicitySerial(context.Orcanodes, unfoundNode.DataplicitySerial, out OrcanodeOnlineStatus unfoundNodeStatus);
var oldNode = FindOrcanodeByDataplicitySerial(foundList, unfoundNode.DataplicitySerial, out OrcanodeOnlineStatus unfoundNodeStatus);
if (oldNode != null)
{
oldNode.DataplicityOnline = null;
Expand All @@ -414,7 +427,7 @@ public async static Task UpdateOrcasoundDataAsync(OrcanodeMonitorContext context
try
{
string json = await _httpClient.GetStringAsync(_orcasoundFeedsUrl);
if (json == "")
if (json.IsNullOrEmpty())
{
return;
}
Expand All @@ -429,8 +442,10 @@ public async static Task UpdateOrcasoundDataAsync(OrcanodeMonitorContext context
return;
}

var foundList = context.Orcanodes.ToList();

// Create a list to track what nodes are no longer returned.
var unfoundList = context.Orcanodes.ToList();
var unfoundList = foundList.ToList();

foreach (JsonElement feed in dataArray.EnumerateArray())
{
Expand Down Expand Up @@ -466,11 +481,11 @@ public async static Task UpdateOrcasoundDataAsync(OrcanodeMonitorContext context
}

Orcanode? node = null;
node = FindOrcanodeByOrcasoundFeedId(context.Orcanodes, feedId.ToString());
node = FindOrcanodeByOrcasoundFeedId(foundList, feedId.ToString());
if (node == null)
{
// We didn't used to store the feed ID, only the name, so try again by name.
node = FindOrcanodeByOrcasoundName(context.Orcanodes, orcasoundName);
node = FindOrcanodeByOrcasoundName(foundList, orcasoundName);
}

// See if we can find a node by dataplicity ID, so that if a node
Expand All @@ -479,7 +494,7 @@ public async static Task UpdateOrcasoundDataAsync(OrcanodeMonitorContext context
Orcanode? dataplicityNode = null;
if (!dataplicitySerial.IsNullOrEmpty())
{
dataplicityNode = FindOrcanodeByDataplicitySerial(context.Orcanodes, dataplicitySerial, out OrcanodeOnlineStatus oldStatus);
dataplicityNode = FindOrcanodeByDataplicitySerial(foundList, dataplicitySerial, out OrcanodeOnlineStatus oldStatus);
if (dataplicityNode != null)
{
if (node == null)
Expand Down Expand Up @@ -547,7 +562,7 @@ public async static Task UpdateOrcasoundDataAsync(OrcanodeMonitorContext context
// Mark any remaining unfound nodes as absent.
foreach (var unfoundNode in unfoundList)
{
var oldNode = FindOrcanodeByOrcasoundFeedId(context.Orcanodes, unfoundNode.OrcasoundFeedId);
var oldNode = FindOrcanodeByOrcasoundFeedId(foundList, unfoundNode.OrcasoundFeedId);
if (oldNode != null)
{
oldNode.OrcasoundFeedId = String.Empty;
Expand Down
3 changes: 0 additions & 3 deletions OrcanodeMonitor/Core/PeriodicTasks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)

await Fetcher.UpdateS3DataAsync(context);

#if ORCAHELLO
// OrcaHello is time-consuming to query so do this last.
await Fetcher.UpdateOrcaHelloDataAsync(context);
#endif
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions OrcanodeMonitor/Migrations/OrcanodeMonitorContextModelSnapshot.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
b.Property<DateTime?>("ManifestUpdatedUtc")
.HasColumnType("datetime2");

b.Property<string>("OrcaHelloId")
.IsRequired()
.HasColumnType("nvarchar(max)");

b.Property<string>("OrcasoundFeedId")
.IsRequired()
.HasColumnType("nvarchar(max)");
Expand Down
22 changes: 7 additions & 15 deletions OrcanodeMonitor/Models/Orcanode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public Orcanode()
DataplicityDescription = string.Empty;
DataplicityName = string.Empty;
DataplicitySerial = string.Empty;
OrcaHelloId = string.Empty;
}

#region persisted
Expand Down Expand Up @@ -166,6 +167,11 @@ public Orcanode()
/// </summary>
public bool? OrcasoundVisible { get; set; }

/// <summary>
/// The "id" field from the OrcaHello hydrophones API.
/// </summary>
public string OrcaHelloId { get; set; }

#endregion persisted

#region derived
Expand Down Expand Up @@ -243,22 +249,8 @@ public OrcanodeOnlineStatus DataplicityConnectionStatus {
}
}

#if ORCAHELLO
public string OrcaHelloName
{
get
{
if (DisplayName == null) return string.Empty;

// Any special cases here, since OrcaHello does not support
// node enumeration, nor does it use the same names as
// Dataplicity or Orcasound.net.
if (DisplayName == "Orcasound Lab") return "Haro Strait";
public OrcanodeOnlineStatus OrcaHelloStatus => OrcaHelloId.IsNullOrEmpty() ? OrcanodeOnlineStatus.Absent : OrcanodeOnlineStatus.Online;

return DisplayName;
}
}
#endif
public OrcanodeOnlineStatus OrcasoundStatus
{
get
Expand Down
23 changes: 22 additions & 1 deletion OrcanodeMonitor/Pages/Index.cshtml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<th>SD Card Util.</th>
<th>Orcasound</th>
<th>S3 Stream</th>
<th>OrcaHello</th>
<!--
<th>Last OrcaHello Detection</th>
<th>Confidence</th>
Expand Down Expand Up @@ -85,11 +86,25 @@
else
{
<td style="background-color: @Model.NodeS3BackgroundColor(item)">
<a href="https://open.quiltdata.com/b/streaming-orcasound-net/tree/@item.S3NodeName/" style="color: @Model.NodeS3TextColor(item)" target="_blank">
<a href="https://open.quiltdata.com/b/audio-orcasound-net/tree/@item.S3NodeName/" style="color: @Model.NodeS3TextColor(item)" target="_blank">
@Html.DisplayFor(modelItem => item.S3StreamStatus)
</a>
</td>
}
@if (item.OrcaHelloStatus == Models.OrcanodeOnlineStatus.Absent)
{
<td style="background-color: @Model.NodeOrcaHelloBackgroundColor(item); color: @Model.NodeOrcaHelloTextColor(item)">
@Html.DisplayFor(modelItem => item.OrcaHelloStatus)
</td>
}
else
{
<td style="background-color: @Model.NodeOrcaHelloBackgroundColor(item)">
<a href="https://open.quiltdata.com/b/audio-orcasound-net/tree/@item.S3NodeName/" style="color: @Model.NodeOrcaHelloTextColor(item)" target="_blank">
@Html.DisplayFor(modelItem => item.OrcaHelloStatus)
</a>
</td>
}
</tr>
}
</table>
Expand Down Expand Up @@ -126,6 +141,12 @@
<li>
<b>S3 Stream Unintelligible</b>: Audio stream is being sent to S3 but it appears to be bad.
</li>
<li>
<b>OrcaHello Absent</b>: OrcaHello does not know about the node.
</li>
<li>
<b>OrcaHello Online</b>: OrcaHello knows about the node.
</li>
</ul>
<p/>
<h1 class="display-4"><a href="/api/ifttt/v1/triggers/nodestateevents">Recent State Events</a></h1>
Expand Down
Loading