From f630ab894e47352729bc5ce8f2cbc0c0926be8d7 Mon Sep 17 00:00:00 2001 From: avosskuehler Date: Thu, 16 Dec 2021 23:17:37 +0100 Subject: [PATCH] BUGFIX: FPS Changes in Video Info Dialog were not reflected in Data Acquisition. BETTER: Video Playback on very fast and very slow fps videos stays at correct timing and does not have deadlocks. Left to solve: Some videos (e.g. interlaced mov) did not show correct timeline calibration. BETTER: No arbitrary motion detection on startup with loaded movie. --- VianaNET/Data/ProcessingData.cs | 153 +++++++++--------- VianaNET/Modules/Video/Control/VideoBase.cs | 134 ++++++++------- VianaNET/Modules/Video/Control/VideoPlayer.cs | 10 +- .../Video/Dialogs/VideoInfoDialog.xaml.cs | 4 +- VianaNET/Modules/Video/VideoWindow.xaml.cs | 10 +- VianaNET/VianaNET.csproj | 40 +++-- VianaNET/app.config | 2 +- VianaNET/packages.config | 18 +-- 8 files changed, 214 insertions(+), 157 deletions(-) diff --git a/VianaNET/Data/ProcessingData.cs b/VianaNET/Data/ProcessingData.cs index a748b09..7c3d817 100644 --- a/VianaNET/Data/ProcessingData.cs +++ b/VianaNET/Data/ProcessingData.cs @@ -274,6 +274,8 @@ public ProcessingData() this.PositiveContrast.CollectionChanged += this.MotionDetectionParameterCollectionChanged; this.PropertyChanged += this.ProcessingDataPropertyChanged; + + this.IsDetectionActivated = this.IsUsingMotionDetection || (this.IsUsingColorDetection && this.IsTargetColorSet); } @@ -557,94 +559,97 @@ public bool ProcessImage() this.watch.Start(); long start = this.watch.ElapsedMilliseconds; - for (int i = 0; i < App.Project.ProcessingData.NumberOfTrackedObjects; i++) + if (App.Project.ProcessingData.IsDetectionActivated) { - // Console.Write("BeforeColorFilte "); - // Console.WriteLine(watch.ElapsedMilliseconds.ToString()); - if (this.ColorThreshold.Count <= i || this.BlobMinDiameter.Count <= i - || this.BlobMaxDiameter.Count <= i || this.CurrentBlobCenter.Count <= i) + for (int i = 0; i < App.Project.ProcessingData.NumberOfTrackedObjects; i++) { - break; - } - - // Get original picture - Video.Instance.RefreshProcessingMap(); + // Console.Write("BeforeColorFilte "); + // Console.WriteLine(watch.ElapsedMilliseconds.ToString()); + if (this.ColorThreshold.Count <= i || this.BlobMinDiameter.Count <= i + || this.BlobMaxDiameter.Count <= i || this.CurrentBlobCenter.Count <= i) + { + break; + } - if (this.IsUsingColorDetection) - { - // Apply color and crop filter if applicable - this.colorAndCropFilter.TargetColor = this.TargetColor.Count > i ? this.TargetColor[i] : Colors.Black; - this.colorAndCropFilter.Threshold = this.ColorThreshold[i]; - this.colorAndCropFilter.ProcessInPlace(Video.Instance.VideoElement.ColorProcessingMapping); - } - else - { - // Only apply crop filter - this.cropFilter.ProcessInPlace(Video.Instance.VideoElement.ColorProcessingMapping); - } + // Get original picture + Video.Instance.RefreshProcessingMap(); - // Apply motion detection if applicable - if (this.IsUsingMotionDetection) - { - if (this.detector.MotionDetectionAlgorithm is TwoFramesDifferenceDetectorSpecial algorithm) + if (this.IsUsingColorDetection) + { + // Apply color and crop filter if applicable + this.colorAndCropFilter.TargetColor = this.TargetColor.Count > i ? this.TargetColor[i] : Colors.Black; + this.colorAndCropFilter.Threshold = this.ColorThreshold[i]; + this.colorAndCropFilter.ProcessInPlace(Video.Instance.VideoElement.ColorProcessingMapping); + } + else { - algorithm.DifferenceThreshold = App.Project.ProcessingData.MotionThreshold[i]; - algorithm.IsPositiveThreshold = App.Project.ProcessingData.PositiveContrast[i]; - algorithm.SuppressNoise = App.Project.ProcessingData.SuppressNoise[i]; + // Only apply crop filter + this.cropFilter.ProcessInPlace(Video.Instance.VideoElement.ColorProcessingMapping); } - Video.Instance.VideoElement.CopyProcessingMapToUnmanagedImage(); - this.detector.ProcessFrame(Video.Instance.VideoElement.UnmanagedImage); - Video.Instance.VideoElement.CopyProcessedDataToProcessingMap(); - } + // Apply motion detection if applicable + if (this.IsUsingMotionDetection) + { + if (this.detector.MotionDetectionAlgorithm is TwoFramesDifferenceDetectorSpecial algorithm) + { + algorithm.DifferenceThreshold = App.Project.ProcessingData.MotionThreshold[i]; + algorithm.IsPositiveThreshold = App.Project.ProcessingData.PositiveContrast[i]; + algorithm.SuppressNoise = App.Project.ProcessingData.SuppressNoise[i]; + } + + Video.Instance.VideoElement.CopyProcessingMapToUnmanagedImage(); + this.detector.ProcessFrame(Video.Instance.VideoElement.UnmanagedImage); + Video.Instance.VideoElement.CopyProcessedDataToProcessingMap(); + } - // Send modified image to blobs control - Video.Instance.VideoElement.UpdateProcessedImageSource(); + // Send modified image to blobs control + Video.Instance.VideoElement.UpdateProcessedImageSource(); - // Get blobs from filtered process - IntPtr mapToUse; - if (this.IsUsingColorDetection && !this.IsUsingMotionDetection) - { - mapToUse = Video.Instance.VideoElement.ColorProcessingMapping; - } - else - { - mapToUse = Video.Instance.VideoElement.MotionProcessingMapping; - } + // Get blobs from filtered process + IntPtr mapToUse; + if (this.IsUsingColorDetection && !this.IsUsingMotionDetection) + { + mapToUse = Video.Instance.VideoElement.ColorProcessingMapping; + } + else + { + mapToUse = Video.Instance.VideoElement.MotionProcessingMapping; + } - Histogram histogram = this.histogrammFilter.FromIntPtrMap(mapToUse); - this.segmentator.Histogram = histogram; - this.segmentator.ThresholdLuminance = histogram.Max * 0.5f; - this.segmentator.MinDiameter = this.BlobMinDiameter[i]; - this.segmentator.MaxDiameter = this.BlobMaxDiameter[i]; + Histogram histogram = this.histogrammFilter.FromIntPtrMap(mapToUse); + this.segmentator.Histogram = histogram; + this.segmentator.ThresholdLuminance = histogram.Max * 0.5f; + this.segmentator.MinDiameter = this.BlobMinDiameter[i]; + this.segmentator.MaxDiameter = this.BlobMaxDiameter[i]; - Segment foundSegment = this.segmentator.Process(); - while (this.DetectedBlob.Count <= i) - { - this.DetectedBlob.Add(new Segment()); - } + Segment foundSegment = this.segmentator.Process(); + while (this.DetectedBlob.Count <= i) + { + this.DetectedBlob.Add(new Segment()); + } - this.DetectedBlob[i] = foundSegment; + this.DetectedBlob[i] = foundSegment; - // Console.Write("AfterBlobDetection: "); - // Console.WriteLine(watch.ElapsedMilliseconds.ToString()); - if (foundSegment.Diagonal != 0 && (foundSegment.Height < (this.colorAndCropFilter.ImageHeight - 10)) - && (foundSegment.Width < (this.colorAndCropFilter.ImageWidth - 10))) - { - this.CurrentBlobCenter[i] = new Point(foundSegment.Center.X, foundSegment.Center.Y); - objectsFound = true; - } - else - { - this.CurrentBlobCenter[i] = null; - } + // Console.Write("AfterBlobDetection: "); + // Console.WriteLine(watch.ElapsedMilliseconds.ToString()); + if (foundSegment.Diagonal != 0 && (foundSegment.Height < (this.colorAndCropFilter.ImageHeight - 10)) + && (foundSegment.Width < (this.colorAndCropFilter.ImageWidth - 10))) + { + this.CurrentBlobCenter[i] = new Point(foundSegment.Center.X, foundSegment.Center.Y); + objectsFound = true; + } + else + { + this.CurrentBlobCenter[i] = null; + } - if (Video.Instance.IsDataAcquisitionRunning) - { - if (this.CurrentBlobCenter[i].HasValue) + if (Video.Instance.IsDataAcquisitionRunning) { - var flippedPoint = new Point(this.CurrentBlobCenter[i].Value.X, Video.Instance.VideoElement.NaturalVideoHeight - this.CurrentBlobCenter[i].Value.Y); - App.Project.VideoData.AddPoint(i, flippedPoint); + if (this.CurrentBlobCenter[i].HasValue) + { + var flippedPoint = new Point(this.CurrentBlobCenter[i].Value.X, Video.Instance.VideoElement.NaturalVideoHeight - this.CurrentBlobCenter[i].Value.Y); + App.Project.VideoData.AddPoint(i, flippedPoint); + } } } } @@ -810,7 +815,7 @@ private void ProcessingDataPropertyChanged(object sender, PropertyChangedEventAr } else if (e.PropertyName == "IsUsingMotionDetection" || e.PropertyName == "IsUsingColorDetection" || e.PropertyName == "IsTargetColorSet") { - this.IsDetectionActivated = this.IsUsingMotionDetection || this.IsUsingColorDetection; + this.IsDetectionActivated = this.IsUsingMotionDetection || (this.IsUsingColorDetection && this.IsTargetColorSet); this.detector.Reset(); Video.Instance.RefreshProcessingMap(); Video.Instance.VideoElement.CopyProcessingMapToUnmanagedImage(); diff --git a/VianaNET/Modules/Video/Control/VideoBase.cs b/VianaNET/Modules/Video/Control/VideoBase.cs index 2ddef7e..c8e705b 100644 --- a/VianaNET/Modules/Video/Control/VideoBase.cs +++ b/VianaNET/Modules/Video/Control/VideoBase.cs @@ -740,95 +740,121 @@ private void Worker_DoWork(object sender, DoWorkEventArgs e) { BackgroundWorker worker = (BackgroundWorker)sender; Stopwatch watch = new Stopwatch(); + //Stopwatch fpswatch = new Stopwatch(); watch.Start(); + //fpswatch.Start(); double frametimeInMS = 41; + double framerateFactor = 1; + double selectionEnd = 1; this.Dispatcher.Invoke(() => { frametimeInMS = this.FrameTimeInMS; + framerateFactor = App.Project.VideoData.FramerateFactor; + selectionEnd = App.Project.VideoData.SelectionEnd; }); - long starttime = 0; - Mat frameMat = new Mat(); - while (!worker.CancellationPending) + using (Mat frameMat = new Mat()) { - var currentPosInMS = this.OpenCVObject.Get(VideoCaptureProperties.PosMsec); - - bool endreached = false; - this.Dispatcher.Invoke(() => + while (!worker.CancellationPending) { - if (currentPosInMS > App.Project.VideoData.SelectionEnd) + var currentPosInMS = this.OpenCVObject.Get(VideoCaptureProperties.PosMsec); + //var currentPosInMS = this.OpenCVObject.Get(VideoCaptureProperties.PosFrames) * frametimeInMS; + + var scaledCurrentPosInMS = currentPosInMS * framerateFactor; + + if (scaledCurrentPosInMS >= selectionEnd) { - endreached = true; + break; } - }); - if (endreached) - { - break; - } - - //using (Mat frameMat = new Mat()) - { - this.OpenCVObject.Read(frameMat); - if (frameMat.Empty()) + if (Video.Instance.VideoMode == VideoMode.File) { - this.Dispatcher.Invoke(() => + // Get the time it took to process the last frame + var total = watch.ElapsedMilliseconds; + + if (total < frametimeInMS) { - this.Stop(); - if (Video.Instance.VideoMode == VideoMode.File) + // Processing time is shorter that default frametime, so wait till frametime is over, to have correct FPS + var wait = new TimeSpan((long)((frametimeInMS - total) * 10000)); + Thread.Sleep(wait); + //Console.WriteLine("In Time, wait: {0}", wait.Milliseconds); + } + else + { + // Processing has taken more time, than a frame should last, so skip the next frame to be in time again. + var skipCount = Math.Floor(total / frametimeInMS); + //Console.WriteLine("Skip: {0}", skipCount); + for (int i = 0; i < skipCount; i++) { - var lastFrameIndex = this.OpenCVObject.Get(VideoCaptureProperties.FrameCount); - this.OpenCVObject.Set(VideoCaptureProperties.PosFrames, lastFrameIndex); - Video.Instance.VideoPlayerElement.RaiseFileComplete(); this.OpenCVObject.Grab(); - this.GrabCurrentFrame(); + this.frameCounter++; } - }); - break; + watch.Restart(); + continue; + } + + watch.Restart(); } - if (this.rotation.HasValue) + // Output FPS + //Console.WriteLine("FPS: {0}", 1f / fpswatch.ElapsedMilliseconds * 1000); + //fpswatch.Restart(); + + // Bildverarbeitung grab, retreive, analyze, send to processing chain { - using (Mat rotMat = new Mat()) + this.OpenCVObject.Read(frameMat); + if (frameMat.Empty()) { - Cv2.Rotate(frameMat, rotMat, this.rotation.Value); + // Letztes Bild erreicht + this.Dispatcher.Invoke(() => + { + this.Stop(); + if (Video.Instance.VideoMode == VideoMode.File) + { + var lastFrameIndex = this.OpenCVObject.Get(VideoCaptureProperties.FrameCount); + this.OpenCVObject.Set(VideoCaptureProperties.PosFrames, lastFrameIndex); + Video.Instance.VideoPlayerElement.RaiseFileComplete(); + this.OpenCVObject.Grab(); + this.GrabCurrentFrame(); + } + }); + break; + } + + if (this.rotation.HasValue) + { + using (Mat rotMat = new Mat()) + { + Cv2.Rotate(frameMat, rotMat, this.rotation.Value); + + // Must create and use WriteableBitmap in the same thread(UI Thread). + this.Dispatcher.Invoke(() => + { + WriteableBitmap newFrame = rotMat.ToWriteableBitmap(); + Video.Instance.OriginalImageSource = newFrame; + Video.Instance.VideoElement.NewFrameCallback(newFrame); + }); + } + } + else + { // Must create and use WriteableBitmap in the same thread(UI Thread). this.Dispatcher.Invoke(() => { - WriteableBitmap newFrame = rotMat.ToWriteableBitmap(); + WriteableBitmap newFrame = frameMat.ToWriteableBitmap(); Video.Instance.OriginalImageSource = newFrame; Video.Instance.VideoElement.NewFrameCallback(newFrame); }); } } - else - { - // Must create and use WriteableBitmap in the same thread(UI Thread). - this.Dispatcher.Invoke(() => - { - WriteableBitmap newFrame = frameMat.ToWriteableBitmap(); - Video.Instance.OriginalImageSource = newFrame; - Video.Instance.VideoElement.NewFrameCallback(newFrame); - }); - } - } - if (Video.Instance.VideoMode == VideoMode.File) - { - while (watch.ElapsedMilliseconds - starttime < frametimeInMS) - { - Thread.Sleep(1); - } - } - starttime = watch.ElapsedMilliseconds; - GC.Collect(); + GC.Collect(); + } } - - frameMat.Dispose(); } /// diff --git a/VianaNET/Modules/Video/Control/VideoPlayer.cs b/VianaNET/Modules/Video/Control/VideoPlayer.cs index db4969e..26ea9ca 100644 --- a/VianaNET/Modules/Video/Control/VideoPlayer.cs +++ b/VianaNET/Modules/Video/Control/VideoPlayer.cs @@ -89,13 +89,18 @@ public override double MediaPositionInMS { get { + // PosMsec in OpenCV is not reliable, https://github.com/opencv/opencv/issues/9053#issuecomment-745635554 var pos = this.OpenCVObject.Get(VideoCaptureProperties.PosMsec); - return pos; + return pos * App.Project.VideoData.FramerateFactor; + + //var pos = this.OpenCVObject.Get(VideoCaptureProperties.PosFrames); + //return pos * this.FrameTimeInMS * App.Project.VideoData.FramerateFactor; } set { - this.OpenCVObject.Set(VideoCaptureProperties.PosMsec, value); + this.OpenCVObject.Set(VideoCaptureProperties.PosMsec, value / App.Project.VideoData.FramerateFactor); + //this.OpenCVObject.Set(VideoCaptureProperties.PosFrames, value / this.FrameTimeInMS / App.Project.VideoData.FramerateFactor); this.OpenCVObject.Grab(); this.GrabCurrentFrame(); this.UpdateFrameIndex(); @@ -299,6 +304,7 @@ public override void Revert() // Seek to the beginning this.OpenCVObject.Set(VideoCaptureProperties.PosMsec, zeroPosition); + //this.OpenCVObject.Set(VideoCaptureProperties.PosFrames, zeroPosition / FrameTimeInMS); this.OpenCVObject.Grab(); this.GrabCurrentFrame(); this.UpdateFrameIndex(); diff --git a/VianaNET/Modules/Video/Dialogs/VideoInfoDialog.xaml.cs b/VianaNET/Modules/Video/Dialogs/VideoInfoDialog.xaml.cs index 48220df..e26d282 100644 --- a/VianaNET/Modules/Video/Dialogs/VideoInfoDialog.xaml.cs +++ b/VianaNET/Modules/Video/Dialogs/VideoInfoDialog.xaml.cs @@ -262,8 +262,8 @@ private void OkClick(object sender, RoutedEventArgs e) { double factor = this.DefaultFrameRate / this.FrameRate; App.Project.VideoData.FramerateFactor = factor; - Control.Video.Instance.VideoPlayerElement.MediaDurationInMS = this.Duration * factor; - Control.Video.Instance.VideoElement.FrameTimeInMS = 1000d / this.FrameRate; + Video.Instance.VideoPlayerElement.MediaDurationInMS = this.Duration * factor; + Video.Instance.VideoElement.FrameTimeInMS = 1000d / this.FrameRate; } this.Close(); diff --git a/VianaNET/Modules/Video/VideoWindow.xaml.cs b/VianaNET/Modules/Video/VideoWindow.xaml.cs index ec0d6ac..20682c2 100644 --- a/VianaNET/Modules/Video/VideoWindow.xaml.cs +++ b/VianaNET/Modules/Video/VideoWindow.xaml.cs @@ -663,6 +663,11 @@ private void LineMouseMove(object sender, MouseEventArgs e) private void OnVideoFrameChanged(object sender, EventArgs e) { //var pos = Video.Instance.VideoElement.OpenCVObject.Get(OpenCvSharp.VideoCaptureProperties.PosMsec); + //if (Video.Instance.VideoMode == VideoMode.File) + //{ + // this.TimelineSlider.Value = pos * App.Project.VideoData.FramerateFactor; + //} + var posFrames = Video.Instance.VideoElement.OpenCVObject.Get(OpenCvSharp.VideoCaptureProperties.PosFrames); if (Video.Instance.VideoMode == VideoMode.File) { @@ -1126,7 +1131,10 @@ private void BtnSetZeroTimeClick(object sender, RoutedEventArgs e) private void TimelineSlider_DragDelta(object sender, DragDeltaEventArgs e) { - Video.Instance.VideoPlayerElement.MediaPositionInMS = this.TimelineSlider.Value; + if (Video.Instance.VideoPlayerElement.CurrentState != VideoBase.PlayState.Running) + { + Video.Instance.VideoPlayerElement.MediaPositionInMS = this.TimelineSlider.Value; + } } } } \ No newline at end of file diff --git a/VianaNET/VianaNET.csproj b/VianaNET/VianaNET.csproj index bc572b0..8d8c7a7 100644 --- a/VianaNET/VianaNET.csproj +++ b/VianaNET/VianaNET.csproj @@ -1,6 +1,6 @@  - + Debug @@ -133,13 +133,13 @@ True - ..\packages\OpenCvSharp4.4.5.3.20210817\lib\net461\OpenCvSharp.dll + ..\packages\OpenCvSharp4.4.5.3.20211207\lib\net461\OpenCvSharp.dll - ..\packages\OpenCvSharp4.4.5.3.20210817\lib\net461\OpenCvSharp.Extensions.dll + ..\packages\OpenCvSharp4.4.5.3.20211207\lib\net461\OpenCvSharp.Extensions.dll - ..\packages\OpenCvSharp4.WpfExtensions.4.5.3.20210817\lib\net461\OpenCvSharp.WpfExtensions.dll + ..\packages\OpenCvSharp4.WpfExtensions.4.5.3.20211207\lib\net461\OpenCvSharp.WpfExtensions.dll ..\packages\OxyPlot.Core.2.1.0\lib\net45\OxyPlot.dll @@ -173,8 +173,8 @@ - - ..\packages\System.Drawing.Common.5.0.2\lib\net461\System.Drawing.Common.dll + + ..\packages\System.Drawing.Common.6.0.0\lib\net461\System.Drawing.Common.dll @@ -185,8 +185,8 @@ ..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll - - ..\packages\System.Runtime.CompilerServices.Unsafe.5.0.0\lib\net45\System.Runtime.CompilerServices.Unsafe.dll + + ..\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll @@ -216,17 +216,29 @@ ..\packages\WPFFolderBrowser.1.0.2\lib\WPFFolderBrowser.dll - ..\packages\WPFLocalizeExtension.3.9.0\lib\net452\WPFLocalizeExtension.dll + ..\packages\WPFLocalizeExtension.3.9.1\lib\net452\WPFLocalizeExtension.dll False ..\..\WpfMath\wpf-math\src\WpfMath\bin\Release\net452\WpfMath.dll - - ..\packages\XAMLMarkupExtensions.2.1.1\lib\net48\XAMLMarkupExtensions.dll + + ..\packages\XAMLMarkupExtensions.2.1.2\lib\net48\XAMLMarkupExtensions.dll - - ..\packages\Extended.Wpf.Toolkit.4.1.0\lib\net40\Xceed.Wpf.Toolkit.dll + + ..\packages\Extended.Wpf.Toolkit.4.2.0\lib\net40\Xceed.Wpf.AvalonDock.dll + + + ..\packages\Extended.Wpf.Toolkit.4.2.0\lib\net40\Xceed.Wpf.AvalonDock.Themes.Aero.dll + + + ..\packages\Extended.Wpf.Toolkit.4.2.0\lib\net40\Xceed.Wpf.AvalonDock.Themes.Metro.dll + + + ..\packages\Extended.Wpf.Toolkit.4.2.0\lib\net40\Xceed.Wpf.AvalonDock.Themes.VS2010.dll + + + ..\packages\Extended.Wpf.Toolkit.4.2.0\lib\net40\Xceed.Wpf.Toolkit.dll @@ -957,6 +969,6 @@ Dieses Projekt verweist auf mindestens ein NuGet-Paket, das auf diesem Computer fehlt. Verwenden Sie die Wiederherstellung von NuGet-Paketen, um die fehlenden Dateien herunterzuladen. Weitere Informationen finden Sie unter "http://go.microsoft.com/fwlink/?LinkID=322105". Die fehlende Datei ist "{0}". - + \ No newline at end of file diff --git a/VianaNET/app.config b/VianaNET/app.config index 1794658..72a398b 100644 --- a/VianaNET/app.config +++ b/VianaNET/app.config @@ -12,7 +12,7 @@ - + diff --git a/VianaNET/packages.config b/VianaNET/packages.config index 592927e..a6f26fa 100644 --- a/VianaNET/packages.config +++ b/VianaNET/packages.config @@ -5,15 +5,15 @@ - + - - - - + + + + @@ -23,12 +23,12 @@ - + - + - - + + \ No newline at end of file