From 3d1b26349bc8f7314ccf950d700707355c2ca1c8 Mon Sep 17 00:00:00 2001 From: towsey Date: Sat, 7 Mar 2020 09:52:13 +1000 Subject: [PATCH] Fixing bugs in merging of acoustic events Issue #300 The main difficulty here is confusing terminology of temporal fileds and properties. Changed some of these to be more explicit. There appears to remain one more bug - the number of acoustic events in the ae.csv file does not match number that appears in the spectrogram images?? --- src/AnalysisBase/ResultBases/EventBase.cs | 17 +- .../Recognizers/Base/WhistleParameters.cs | 37 +--- .../Recognizers/GenericRecognizer.cs | 2 +- src/AudioAnalysisTools/AcousticEvent.cs | 191 +++++++++--------- 4 files changed, 114 insertions(+), 133 deletions(-) diff --git a/src/AnalysisBase/ResultBases/EventBase.cs b/src/AnalysisBase/ResultBases/EventBase.cs index ea37af352..42f4fd6d2 100644 --- a/src/AnalysisBase/ResultBases/EventBase.cs +++ b/src/AnalysisBase/ResultBases/EventBase.cs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------------------------------- // // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -12,14 +12,14 @@ namespace AnalysisBase.ResultBases using System; /// - /// The base class for all Event style results + /// The base class for all Event style results. /// public abstract class EventBase : ResultBase { private double eventStartSeconds; /// - /// Gets or sets the time the current audio segment is offset from the start of the file/recording. + /// Gets or sets the time (in seconds) from start of the file/recording to start of the current audio segment. /// /// /// will always be greater than or equal to . @@ -36,7 +36,7 @@ public abstract class EventBase : ResultBase /// /// /// 2017-09: This field USED to be offset relative to the current segment. - /// 2017-09: This field is NOW equivalent to + /// 2017-09: This field is NOW equivalent to . /// public virtual double EventStartSeconds { @@ -60,10 +60,13 @@ public virtual double EventStartSeconds /// public virtual double? LowFrequencyHertz { get; protected set; } - protected void SetEventStartRelative(TimeSpan segmentStart, double eventStartSegmentRelative) + /// + /// Sets both the Segment start and the Event start wrt to recording. + /// + protected void SetSegmentAndEventStartsWrtRecording(TimeSpan segmentStartWrtRecording, double eventStartWrtSegment) { - this.SegmentStartSeconds = segmentStart.TotalSeconds; - this.EventStartSeconds = this.SegmentStartSeconds + eventStartSegmentRelative; + this.SegmentStartSeconds = segmentStartWrtRecording.TotalSeconds; + this.EventStartSeconds = this.SegmentStartSeconds + eventStartWrtSegment; } } } diff --git a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs index 071b49772..f1007d972 100644 --- a/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs +++ b/src/AnalysisPrograms/Recognizers/Base/WhistleParameters.cs @@ -30,7 +30,7 @@ public static (List, double[]) GetWhistles( double decibelThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartOffset) + TimeSpan segmentStartWrtRecording) { var sonogramData = sonogram.Data; int frameCount = sonogramData.GetLength(0); @@ -39,10 +39,6 @@ public static (List, double[]) GetWhistles( double binWidth = nyquist / (double)binCount; int minBin = (int)Math.Round(minHz / binWidth); int maxBin = (int)Math.Round(maxHz / binWidth); - //int binCountInBand = maxBin - minBin + 1; - - // buffer zone around whistle is four bins wide. - int N = 4; // list of accumulated acoustic events var events = new List(); @@ -54,7 +50,8 @@ public static (List, double[]) GetWhistles( // set up an intensity array for the frequency bin. double[] intensity = new double[frameCount]; - if (minBin < N) + // buffer zone around whistle is four bins wide. + if (minBin < 4) { // for all time frames in this frequency bin for (int t = 0; t < frameCount; t++) @@ -96,7 +93,7 @@ public static (List, double[]) GetWhistles( decibelThreshold, minDuration, maxDuration, - segmentStartOffset); + segmentStartWrtRecording); // add to conbined intensity array for (int t = 0; t < frameCount; t++) @@ -110,33 +107,9 @@ public static (List, double[]) GetWhistles( } //end for all freq bins // combine adjacent acoustic events - events = AcousticEvent.CombineOverlappingEvents(events); + events = AcousticEvent.CombineOverlappingEvents(events, segmentStartWrtRecording); return (events, combinedIntensityArray); } - - /* - /// - /// Calculates the average intensity in a freq band having min and max freq, - /// AND then subtracts average intensity in the side/buffer bands, below and above. - /// THis method adds dB log values incorrectly but it is faster than doing many log conversions. - /// This method is used to find acoustic events and is accurate enough for the purpose. - /// - public static double[] CalculateFreqBandAvIntensityMinusBufferIntensity(double[,] sonogramData, int minHz, int maxHz, int nyquist) - { - var bandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, minHz, maxHz, nyquist); - var bottomSideBandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, minHz - bottomHzBuffer, minHz, nyquist); - var topSideBandIntensity = SNR.CalculateFreqBandAvIntensity(sonogramData, maxHz, maxHz + topHzBuffer, nyquist); - - int frameCount = sonogramData.GetLength(0); - double[] netIntensity = new double[frameCount]; - for (int i = 0; i < frameCount; i++) - { - netIntensity[i] = bandIntensity[i] - bottomSideBandIntensity[i] - topSideBandIntensity[i]; - } - - return netIntensity; - } - */ } } diff --git a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs index 76c451136..d5c77f897 100644 --- a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs +++ b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs @@ -334,7 +334,7 @@ static void SaveDebugSpectrogram(RecognizerResults results, Config genericConfig var image3 = SpectrogramTools.GetSonogramPlusCharts(results.Sonogram, results.Events, results.Plots, null); //image3.Save(Path.Combine(outputDirectory.FullName, baseName + ".profile.png")); - image3.Save(Path.Combine("C:\\temp", baseName + ".profile.png")); + //image3.Save(Path.Combine("C:\\temp", baseName + ".profile.png")); //sonogram.GetImageFullyAnnotated("test").Save("C:\\temp\\test.png"); } diff --git a/src/AudioAnalysisTools/AcousticEvent.cs b/src/AudioAnalysisTools/AcousticEvent.cs index e2e726814..f13354e13 100644 --- a/src/AudioAnalysisTools/AcousticEvent.cs +++ b/src/AudioAnalysisTools/AcousticEvent.cs @@ -83,30 +83,39 @@ public AcousticEventClassMap() public double TimeStart { get; private set; } /// - /// Gets or sets units = seconds - /// Time offset from start of current segment to end of the event. + /// Gets the time offset (in seconds) from start of current segment to end of the event. /// Written into the csv file under column "EventEndSeconds" /// This field is NOT in EventBase. EventBase only requires TimeStart because it is designed to also accomodate points. /// /// - /// Note: converted to private setter so we can control how this is set. Recommend using - /// after event instantiation to modify bounds. + /// Note: converted to private setter so we can control how this is set. + /// Recommend using after event instantiation to modify bounds. /// public double TimeEnd { get; private set; } + /// + /// Gets the end time of an event WRT the recording/file start. + /// public double EventEndSeconds => this.TimeEnd + this.SegmentStartSeconds; + /// + /// Gets the start time of an event WRT the recording/file start. + /// public override double EventStartSeconds => this.TimeStart + this.SegmentStartSeconds; + /// + /// Set the start and end times of an event with respect to the segment start time + /// AND also calls method to set event start time with respect the recording/file start. + /// public void SetEventPositionRelative( - TimeSpan segmentStartOffset, - double eventStartSegment, - double eventEndSegment) + TimeSpan segmentStartWrtRecording, + double eventStartWrtSegment, + double eventEndWrtSegment) { - this.TimeStart = eventStartSegment; - this.TimeEnd = eventEndSegment; + this.TimeStart = eventStartWrtSegment; + this.TimeEnd = eventEndWrtSegment; - this.SetEventStartRelative(segmentStartOffset, eventStartSegment); + this.SetSegmentAndEventStartsWrtRecording(segmentStartWrtRecording, eventStartWrtSegment); } /// @@ -126,30 +135,39 @@ public void SetEventPositionRelative( } } - /// Gets or sets units = Hertz + /// Gets or sets units = Hertz. public double HighFrequencyHertz { get; set; } + /// + /// Gets the bandwidth of an acoustic event. + /// public double Bandwidth => this.HighFrequencyHertz - this.LowFrequencyHertz + 1; public bool IsMelscale { get; set; } + /// + /// Gets or sets the bounds of an event with respect to the segment start + /// BUT in terms of the frame count (from segment start) and frequency bin (from zero Hertz). + /// This is no longer the preferred way to operate with acoustic event bounds. + /// Better to use real units (seconds and Hertz) and provide the acoustic event with scale information. + /// public Oblong Oblong { get; set; } - /// Gets or sets required for conversions to & from MEL scale AND for drawing event on spectrum + /// Gets or sets required for conversions to & from MEL scale AND for drawing event on spectrum. public int FreqBinCount { get; set; } /// - /// Gets required for freq-binID conversions + /// Gets required for freq-binID conversions. /// public double FreqBinWidth { get; private set; } - /// Gets frame duration in seconds + /// Gets frame duration in seconds. public double FrameDuration { get; private set; } - /// Gets or sets time between frame starts in seconds. Inverse of FramesPerSecond + /// Gets or sets time between frame starts in seconds. Inverse of FramesPerSecond. public double FrameOffset { get; set; } - /// Gets or sets number of frame starts per second. Inverse of the frame offset + /// Gets or sets number of frame starts per second. Inverse of the frame offset. public double FramesPerSecond { get; set; } //PROPERTIES OF THE EVENTS i.e. Name, SCORE ETC @@ -162,7 +180,7 @@ public void SetEventPositionRelative( /// Gets or sets average score through the event. public string ScoreComment { get; set; } - /// Gets or sets score normalised in range [0,1]. NOTE: Max is set = to five times user supplied threshold + /// Gets or sets score normalised in range [0,1]. NOTE: Max is set = to five times user supplied threshold. public double ScoreNormalised { get; set; } /// Gets max Possible Score: set = to 5x user supplied threshold. An arbitrary value used for score normalisation. @@ -174,8 +192,10 @@ public void SetEventPositionRelative( public string Score2Name { get; set; } - /// Gets or sets second score if required - public double Score2 { get; set; } // e.g. for Birgits recognisers + /// Gets or sets second score if required. + /// Was used for Birgits recognisers but should now be used only for debug purposes. + /// + public double Score2 { get; set; } /// /// Gets or sets a list of points that can be used to identifies features in spectrogram relative to the Event. @@ -184,7 +204,11 @@ public void SetEventPositionRelative( /// public List Points { get; set; } - public double Periodicity { get; set; } // for events which have an oscillating acoustic energy - used for frog calls + /// + /// Gets or sets the periodicity of acoustic energy in an event. + /// Use for events which have an oscillating acoustic energy - e.g. for frog calls. + /// + public double Periodicity { get; set; } public double DominantFreq { get; set; } // the dominant freq in the event - used for frog calls @@ -192,8 +216,8 @@ public void SetEventPositionRelative( // double I1Var; //, // double I2MeandB; // mean intensity of pixels in the event after Wiener filter, prior to noise subtraction // double I2Var; //, - private double I3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING - private double I3Var; // variance of intensity of pixels in the event. + private double i3Mean; // mean intensity of pixels in the event AFTER noise reduciton - USED FOR CLUSTERING + private double i3Var; // variance of intensity of pixels in the event. // following are no longer needed. Delete eventually. /* @@ -213,10 +237,10 @@ public void SetEventPositionRelative( /// Gets or sets a value indicating whether use this if want to filter or tag some members of a list for some purpose. public bool Tag { get; set; } - /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event intensity + /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event intensity. public int Intensity { get; set; } - /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event quality + /// Gets or sets assigned value when reading in a list of user identified events. Indicates a user assigned assessment of event quality. public int Quality { get; set; } public Color BorderColour { get; set; } @@ -231,11 +255,20 @@ public AcousticEvent() this.IsMelscale = false; } - public AcousticEvent(TimeSpan segmentStartOffset, double startTime, double eventDuration, double minFreq, double maxFreq) + /// + /// Initializes a new instance of the class. + /// This constructor requires the minimum information to establish the temporal and frequency bounds of an acoustic event. + /// + /// The start of the current segment with respect to start of recording/file. + /// event start with respect to start of segment. + /// event end with respect to start of segment. + /// Lower frequency bound of event. + /// Upper frequency bound of event. + public AcousticEvent(TimeSpan segmentStartWrtRecording, double startTimeWrtSegment, double eventDuration, double minFreq, double maxFreq) : this() { - this.SetEventPositionRelative(segmentStartOffset, startTime, startTime + eventDuration); - + var endTimeWrtSegment = startTimeWrtSegment + eventDuration; + this.SetEventPositionRelative(segmentStartWrtRecording, startTimeWrtSegment, endTimeWrtSegment); this.LowFrequencyHertz = minFreq; this.HighFrequencyHertz = maxFreq; @@ -246,6 +279,8 @@ public AcousticEvent(TimeSpan segmentStartOffset, double startTime, double event /// /// Initializes a new instance of the class. /// This constructor currently works ONLY for linear Hertz scale events. + /// It requires the event bounds to provided (using Oblong) in terms of time frame and frequency bin counts. + /// Scale information must also be provided to convert bounds into real values (seconds, Hertz). /// /// An oblong initialized with bin and frame numbers marking location of the event. /// to set the freq scale. @@ -503,12 +538,6 @@ public bool Overlaps(AcousticEvent ae) /// public static double EventFractionalOverlap(AcousticEvent event1, AcousticEvent event2) { - //if (event1.EndTime < event2.StartTime) return 0.0; - //if (event2.EndTime < event1.StartTime) return 0.0; - //if (event1.MaxFreq < event2.MinFreq) return 0.0; - //if (event2.MaxFreq < event1.MinFreq) return 0.0; - //at this point the two events do overlap - int timeOverlap = Oblong.RowOverlap(event1.Oblong, event2.Oblong); if (timeOverlap == 0) { @@ -558,6 +587,19 @@ public static bool EventsOverlap(AcousticEvent event1, AcousticEvent event2) timeOverlap = true; } + // now check possibility that event2 is inside event1. + //check if event 2 starts within event 1 + if (event2.EventStartSeconds >= event1.EventStartSeconds && event2.EventStartSeconds <= event1.EventEndSeconds) + { + timeOverlap = true; + } + + // check if event 2 ends within event 1 + if (event2.EventEndSeconds >= event1.EventStartSeconds && event2.EventEndSeconds <= event1.EventEndSeconds) + { + timeOverlap = true; + } + //check if event 1 freq band overlaps event 2 freq band if (event1.HighFrequencyHertz >= event1.LowFrequencyHertz && event1.HighFrequencyHertz <= event2.HighFrequencyHertz) { @@ -579,7 +621,7 @@ public static bool EventsOverlap(AcousticEvent event1, AcousticEvent event2) /// Freq dimension = bins = matrix columns. Origin is top left - as per matrix in the sonogram class. /// Time dimension = frames = matrix rows. /// - public static List CombineOverlappingEvents(List events) + public static List CombineOverlappingEvents(List events, TimeSpan segmentStartWrtRecording) { if (events.Count < 2) { @@ -592,7 +634,7 @@ public static List CombineOverlappingEvents(List e { if (EventsOverlap(events[i], events[j])) { - events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j]); + events[j] = AcousticEvent.MergeTwoEvents(events[i], events[j], segmentStartWrtRecording); events.RemoveAt(i); break; } @@ -602,14 +644,16 @@ public static List CombineOverlappingEvents(List e return events; } - public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2) + public static AcousticEvent MergeTwoEvents(AcousticEvent e1, AcousticEvent e2, TimeSpan segmentStartWrtRecording) { - //e1.EventEndSeconds = Math.Max(e1.EventEndSeconds, e2.EventEndSeconds); - e1.EventStartSeconds = Math.Min(e1.EventStartSeconds, e2.EventStartSeconds); + //segmentStartOffset = TimeSpan.Zero; + var minTime = Math.Min(e1.TimeStart, e2.TimeStart); + var maxTime = Math.Max(e1.TimeEnd, e2.TimeEnd); + e1.SetEventPositionRelative(segmentStartWrtRecording, minTime, maxTime); e1.LowFrequencyHertz = Math.Min(e1.LowFrequencyHertz, e2.LowFrequencyHertz); e1.HighFrequencyHertz = Math.Max(e1.HighFrequencyHertz, e2.HighFrequencyHertz); - - //e1.ResultMinute = (int)e1.ResultStartSeconds.Floor(); + e1.Score = Math.Max(e1.Score, e2.Score); + e1.ScoreNormalised = Math.Max(e1.ScoreNormalised, e2.ScoreNormalised); e1.ResultStartSeconds = e1.EventStartSeconds; return e1; } @@ -667,8 +711,8 @@ public static void Time2RowIDs(double startTime, double duration, double frameOf public void SetNetIntensityAfterNoiseReduction(double mean, double var) { - this.I3Mean = mean; - this.I3Var = var; + this.i3Mean = mean; + this.i3Var = var; } /// @@ -1248,9 +1292,9 @@ public static List GetEventsAroundMaxima( /// The method uses the passed scoreThreshold in order to calculate a normalised score. /// Max possible score := threshold * 5. /// normalised score := score / maxPossibleScore. - /// Some analysis techniques (e.g. OD) have their own methods for extracting events from score arrays. + /// Some analysis techniques (e.g. Oscillation Detection) have their own methods for extracting events from score arrays. /// - /// the array of scores + /// the array of scores. /// lower freq bound of the acoustic event. /// upper freq bound of the acoustic event. /// the time scale required by AcousticEvent class. @@ -1258,7 +1302,7 @@ public static List GetEventsAroundMaxima( /// threshold. /// duration of event must exceed this to count as an event. /// duration of event must be less than this to count as an event. - /// offset. + /// offset. /// a list of acoustic events. public static List ConvertScoreArray2Events( double[] scores, @@ -1269,14 +1313,14 @@ public static List ConvertScoreArray2Events( double scoreThreshold, double minDuration, double maxDuration, - TimeSpan segmentStartOffset) + TimeSpan segmentStartWrtRecording) { int count = scores.Length; var events = new List(); double maxPossibleScore = 5 * scoreThreshold; // used to calculate a normalised score between 0 - 1.0 bool isHit = false; double frameOffset = 1 / framesPerSec; // frame offset in fractions of second - double startTime = 0.0; + double startTimeWrtSegment = 0.0; // units = seconds int startFrame = 0; // pass over all frames @@ -1286,7 +1330,7 @@ public static List ConvertScoreArray2Events( { //start of an event isHit = true; - startTime = i * frameOffset; + startTimeWrtSegment = i * frameOffset; startFrame = i; } else // check for the end of an event @@ -1295,7 +1339,7 @@ public static List ConvertScoreArray2Events( // this is end of an event, so initialise it isHit = false; double endTime = i * frameOffset; - double duration = endTime - startTime; + double duration = endTime - startTimeWrtSegment; // if (duration < minDuration) continue; //skip events with duration shorter than threshold if (duration < minDuration || duration > maxDuration) @@ -1312,14 +1356,8 @@ public static List ConvertScoreArray2Events( av /= i - startFrame + 1; - //NOTE av cannot be < threhsold because event started and ended based on threhsold. - // Therefore remove the following condition on 04/02/2020 - //if (av < scoreThreshold) - //{ - // continue; //skip events whose score is < the threshold - //} - - AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz); + // Initialize the event. + AcousticEvent ev = new AcousticEvent(segmentStartWrtRecording, startTimeWrtSegment, duration, minHz, maxHz); ev.SetTimeAndFreqScales(framesPerSec, freqBinWidth); ev.Score = av; @@ -1366,12 +1404,12 @@ public static double[] ExtractScoreArrayFromEvents(List events, i double windowOffset = events[0].FrameOffset; double frameRate = 1 / windowOffset; //frames per second - //int count = events.Count; - foreach ( AcousticEvent ae in events) + foreach (AcousticEvent ae in events) { if (!ae.Name.Equals(nameOfTargetEvent)) { - continue; //skip irrelevant events + //skip irrelevant events + continue; } int startFrame = (int)(ae.TimeStart * frameRate); @@ -1388,39 +1426,6 @@ public static double[] ExtractScoreArrayFromEvents(List events, i //############################################################################################################################################## - /// - /// TODO: THis should be deprecated! - /// This method is used to do unit test on lists of events. - /// First developed for frog recognizers - October 2016. - /// - public static void TestToCompareEvents(string fileName, DirectoryInfo opDir, string testName, List events) - { - var testDir = new DirectoryInfo(opDir + $"\\UnitTest_{testName}"); - var benchmarkDir = new DirectoryInfo(testDir + "\\ExpectedOutput"); - if (!benchmarkDir.Exists) - { - benchmarkDir.Create(); - } - - var benchmarkFilePath = Path.Combine(benchmarkDir.FullName, fileName + ".TestEvents.csv"); - var eventsFilePath = Path.Combine(testDir.FullName, fileName + ".Events.csv"); - var eventsFile = new FileInfo(eventsFilePath); - Csv.WriteToCsv(eventsFile, events); - - LoggedConsole.WriteLine($"# EVENTS TEST: Comparing List of {testName} events with those in benchmark file:"); - var benchmarkFile = new FileInfo(benchmarkFilePath); - if (!benchmarkFile.Exists) - { - LoggedConsole.WriteWarnLine(" A file of test/benchmark events does not exist. Writing output as future events-test file"); - Csv.WriteToCsv(benchmarkFile, events); - } - else - { - // compare the test events with benchmark - TestTools.FileEqualityTest("Compare acoustic events.", eventsFile, benchmarkFile); - } - } - /// /// Although not currently used, this method and following methods could be useful in future for clustering of events. ///