From ab121b84dcdbf092430bc1638ddc75c79a6b92b0 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Fri, 11 Oct 2019 16:04:50 +0200 Subject: [PATCH 1/5] Export pts in *_timstamps.csv file --- pupil_src/shared_modules/av_writer.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index 4d0eeae537..bc0a1d0dad 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -55,7 +55,7 @@ """ -def write_timestamps(file_loc, timestamps, output_format="npy"): +def write_timestamps(file_loc, timestamps, output_format="npy", pts=None): """ Attritbutes: output_format (str): Output file format. Available values: @@ -73,7 +73,18 @@ def write_timestamps(file_loc, timestamps, output_format="npy"): if output_format in ("npy", "all"): np.save(ts_loc + ".npy", ts) if output_format in ("csv", "all"): - np.savetxt(ts_loc + ".csv", ts, fmt="%f", header="timestamps [seconds]") + if pts is None: + np.savetxt(ts_loc + ".csv", ts, fmt="%f", header="timestamps [seconds]") + else: + pts = np.asarray(pts) + ts_pts = np.vstack((ts, pts)).T + np.savetxt( + ts_loc + ".csv", + ts_pts, + fmt=["%f", "%i"], + delimiter=",", + header="timestamps [seconds],pts", + ) class AV_Writer(abc.ABC): @@ -87,6 +98,7 @@ def __init__(self, output_file_path: str, start_time_synced: int): """ self.timestamps = [] + self.pts = [] self.start_time = start_time_synced self.last_video_pts = float("-inf") @@ -175,6 +187,7 @@ def write_video_frame(self, input_frame): self.last_video_pts = pts self.timestamps.append(ts) + self.pts.append(pts) def close(self, timestamp_export_format="npy"): """Close writer, triggering stream and timestamp save.""" @@ -190,7 +203,10 @@ def close(self, timestamp_export_format="npy"): if timestamp_export_format is not None: write_timestamps( - self.output_file_path, self.timestamps, timestamp_export_format + self.output_file_path, + self.timestamps, + timestamp_export_format, + pts=self.pts, ) self.container.close() From 042606c85d38e12bf2cd53808027612d8b0187fb Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Mon, 4 Nov 2019 10:35:37 +0100 Subject: [PATCH 2/5] Cleanup imports --- pupil_src/shared_modules/av_writer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index bc0a1d0dad..e41ae2227a 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -9,20 +9,20 @@ ---------------------------------------------------------------------------~(*) """ +import abc + # logging import logging import multiprocessing as mp import os -import platform -from fractions import Fraction -import abc import typing as T +from fractions import Fraction +import av import numpy as np +from av.packet import Packet import audio_utils -import av -from av.packet import Packet logger = logging.getLogger(__name__) From 5c269fa8b75151892f3af6be87853deed9c51ebf Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Mon, 4 Nov 2019 12:02:56 +0100 Subject: [PATCH 3/5] Read pts from exported video --- pupil_src/shared_modules/av_writer.py | 54 +++++++++++++-------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index e41ae2227a..2f02a4bb27 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -23,6 +23,7 @@ from av.packet import Packet import audio_utils +from video_capture.utils import Video logger = logging.getLogger(__name__) @@ -55,7 +56,7 @@ """ -def write_timestamps(file_loc, timestamps, output_format="npy", pts=None): +def write_timestamps(file_loc, timestamps, output_format="npy"): """ Attritbutes: output_format (str): Output file format. Available values: @@ -73,18 +74,19 @@ def write_timestamps(file_loc, timestamps, output_format="npy", pts=None): if output_format in ("npy", "all"): np.save(ts_loc + ".npy", ts) if output_format in ("csv", "all"): - if pts is None: - np.savetxt(ts_loc + ".csv", ts, fmt="%f", header="timestamps [seconds]") - else: - pts = np.asarray(pts) - ts_pts = np.vstack((ts, pts)).T - np.savetxt( - ts_loc + ".csv", - ts_pts, - fmt=["%f", "%i"], - delimiter=",", - header="timestamps [seconds],pts", - ) + output_video = Video(file_loc) + if not output_video.is_valid: + logger.error(f"Failed to extract PTS frome exported video {file_loc}") + return + pts = output_video.pts + ts_pts = np.vstack((ts, pts)).T + np.savetxt( + ts_loc + ".csv", + ts_pts, + fmt=["%f", "%i"], + delimiter=",", + header="timestamps [seconds],pts", + ) class AV_Writer(abc.ABC): @@ -98,7 +100,6 @@ def __init__(self, output_file_path: str, start_time_synced: int): """ self.timestamps = [] - self.pts = [] self.start_time = start_time_synced self.last_video_pts = float("-inf") @@ -173,21 +174,21 @@ def write_video_frame(self, input_frame): # ensure strong monotonic pts pts = max(pts, self.last_video_pts + 1) - n_frames = self.video_stream.frames # TODO: Use custom Frame wrapper class, that wraps backend-specific frames. # This way we could just attach the pts here to the frame. # Currently this will fail e.g. for av.VideoFrame. + video_packed_encoded = False for packet in self.encode_frame(input_frame, pts): + if packet.stream is self.video_stream: + video_packed_encoded = True self.container.mux(packet) - if self.video_stream.frames != n_frames + 1: - logger.warning( - f"Failed to write frame #{n_frames} to video file. Ignoring." - ) + + if not video_packed_encoded: + logger.warning(f"Encoding frame {input_frame.index} failed!") return self.last_video_pts = pts self.timestamps.append(ts) - self.pts.append(pts) def close(self, timestamp_export_format="npy"): """Close writer, triggering stream and timestamp save.""" @@ -201,17 +202,14 @@ def close(self, timestamp_export_format="npy"): for packet in self.video_stream.encode(None): self.container.mux(packet) - if timestamp_export_format is not None: - write_timestamps( - self.output_file_path, - self.timestamps, - timestamp_export_format, - pts=self.pts, - ) - self.container.close() self.closed = True + if timestamp_export_format is not None: + write_timestamps( + self.output_file_path, self.timestamps, timestamp_export_format + ) + def release(self): """Close writer, triggering stream and timestamp save.""" self.close() From 825906785342a4443688a82f8eec426c279d8a38 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Mon, 4 Nov 2019 13:21:00 +0100 Subject: [PATCH 4/5] Only export timestamps if at least one frame was written --- pupil_src/shared_modules/av_writer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index 2f02a4bb27..9e4377c286 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -205,7 +205,9 @@ def close(self, timestamp_export_format="npy"): self.container.close() self.closed = True - if timestamp_export_format is not None: + if self.configured and timestamp_export_format is not None: + # Requires self.container to be closed since we extract pts + # from the exported video file. write_timestamps( self.output_file_path, self.timestamps, timestamp_export_format ) From ac2751413580840904d146ebe4a2b1a9dd87b70f Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Mon, 4 Nov 2019 13:22:55 +0100 Subject: [PATCH 5/5] Add warning regarding 1 frame -> 1 packet assumption --- pupil_src/shared_modules/av_writer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index 9e4377c286..a62de2c08f 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -180,6 +180,10 @@ def write_video_frame(self, input_frame): video_packed_encoded = False for packet in self.encode_frame(input_frame, pts): if packet.stream is self.video_stream: + if video_packed_encoded: + # NOTE: Assumption: Each frame is encoded into a single packet! + # This is required for the frame.pts == packet.pts assumption below. + logger.warning("Single frame yielded more than one packet") video_packed_encoded = True self.container.mux(packet)