[timecode] Add pts and time base properties

Author: Breakthrough

scenedetect/backends/opencv.py

@@ -206,6 +206,8 @@ def timecode(self) -> Timecode:
 
     @property
     def position(self) -> FrameTimecode:
+        # TODO(https://scenedetect.com/issue/168): See if there is a better way to do this, or
+        # add a config option before landing this.
         if _USE_PTS_IN_DEVELOPMENT:
             return FrameTimecode(timecode=self.timecode, fps=self.frame_rate)
         if self.frame_number < 1:
@@ -226,10 +228,7 @@ def seek(self, target: ty.Union[FrameTimecode, float, int]):
         if target < 0:
             raise ValueError("Target seek position cannot be negative!")
 
-        if _USE_PTS_IN_DEVELOPMENT:
-            # TODO(https://scenedetect.com/issue/168): Shouldn't use frames for VFR video here.
-            raise NotImplementedError()
-
+        # TODO(https://scenedetect.com/issue/168): Shouldn't use frames for VFR video here.
         # Have to seek one behind and call grab() after to that the VideoCapture
         # returns a valid timestamp when using CAP_PROP_POS_MSEC.
         target_frame_cv2 = (self.base_timecode + target).frame_num

scenedetect/backends/pyav.py

@@ -12,6 +12,7 @@
 """:class:`VideoStreamAv` provides an adapter for the PyAV av.InputContainer object."""
 
 import typing as ty
+from fractions import Fraction
 from logging import getLogger
 
 import av
@@ -22,7 +23,6 @@
 from scenedetect.video_stream import FrameRateUnavailable, VideoOpenFailure, VideoStream
 
 logger = getLogger("pyscenedetect")
-
 VALID_THREAD_MODES = ["NONE", "SLICE", "FRAME", "AUTO"]
 
 
@@ -36,7 +36,7 @@ class VideoStreamAv(VideoStream):
     def __init__(
         self,
         path_or_io: ty.Union[ty.AnyStr, ty.BinaryIO],
-        framerate: ty.Optional[float] = None,
+        framerate: ty.Optional[ty.Union[float, Fraction]] = None,
         name: ty.Optional[str] = None,
         threading_mode: ty.Optional[str] = None,
         suppress_output: bool = False,
@@ -123,15 +123,14 @@ def __init__(
             )
             if frame_rate is None or frame_rate == 0:
                 raise FrameRateUnavailable()
-            # TODO: Refactor FrameTimecode to support raw timing rather than framerate based calculations.
-            # See https://pyav.org/docs/develop/api/stream.html for details.
-            frame_rate = frame_rate.numerator / float(frame_rate.denominator)
             if frame_rate < MAX_FPS_DELTA:
                 raise FrameRateUnavailable()
-            self._frame_rate: float = frame_rate
+            self._frame_rate: Fraction = frame_rate
         else:
             assert framerate >= MAX_FPS_DELTA
-            self._frame_rate: float = framerate
+            self._frame_rate: Fraction = (
+                framerate if isinstance(framerate, Fraction) else Fraction.from_float(framerate)
+            )
 
         # Calculate duration after we have set the framerate.
         self._duration_frames = self._get_duration()
@@ -212,6 +211,16 @@ def frame_number(self) -> int:
             return self.position.frame_num + 1
         return 0
 
+    @property
+    def rate(self) -> Fraction:
+        return self._video_stream.guessed_rate
+
+    @property
+    def time_base(self) -> Fraction:
+        if self._frame:
+            return self._frame.time_base
+        return None
+
     @property
     def aspect_ratio(self) -> float:
         """Pixel aspect ratio as a float (1.0 represents square pixels)."""
@@ -250,10 +259,7 @@ def seek(self, target: ty.Union[FrameTimecode, float, int]) -> None:
             raise ValueError("Target cannot be negative!")
         beginning = target == 0
 
-        if _USE_PTS_IN_DEVELOPMENT:
-            # TODO(https://scenedetect.com/issue/168): Need to handle PTS here.
-            raise NotImplementedError()
-
+        # TODO(https://scenedetect.com/issues/168): This breaks with PTS mode enabled.
         target = self.base_timecode + target
         if target >= 1:
             target = target - 1

scenedetect/common.py

@@ -172,18 +172,21 @@ def __init__(
             TypeError: Thrown if either `timecode` or `fps` are unsupported types.
             ValueError: Thrown when specifying a negative timecode or framerate.
         """
-        # The following two properties are what is used to keep track of time
-        # in a frame-specific manner.  Note that once the framerate is set,
-        # the value should never be modified (only read if required).
-        # TODO(v1.0): Make these actual @properties.
-        self._framerate: Fraction = None
+        # NOTE: FrameTimecode will have either a `Timecode` representation, a `seconds`
+        # representation, or only a frame number. We cache the calculated values for later use
+        # for the parameters that are missing.
+        self._rate: Fraction = None
+        """Rate at which time passes between frames, measured in frames/sec."""
         self._frame_num = None
+        """Frame number which may be estimated."""
         self._timecode: ty.Optional[Timecode] = None
+        """Presentation timestamp from the backend."""
         self._seconds: ty.Optional[float] = None
+        """An explicit point in time."""
 
         # Copy constructor.
         if isinstance(timecode, FrameTimecode):
-            self._framerate = timecode._framerate if fps is None else fps
+            self._rate = timecode._rate if fps is None else fps
             self._frame_num = timecode._frame_num
             self._timecode = timecode._timecode
             self._seconds = timecode._seconds
@@ -196,15 +199,15 @@ def __init__(
         if fps is None:
             raise TypeError("fps is a required argument.")
         if isinstance(fps, FrameTimecode):
-            self._framerate = fps._framerate
+            self._rate = fps._rate
         elif isinstance(fps, float):
             if fps <= MAX_FPS_DELTA:
                 raise ValueError("Framerate must be positive and greater than zero.")
-            self._framerate = Fraction.from_float(fps)
+            self._rate = Fraction.from_float(fps)
         elif isinstance(fps, Fraction):
             if float(fps) <= MAX_FPS_DELTA:
                 raise ValueError("Framerate must be positive and greater than zero.")
-            self._framerate = fps
+            self._rate = fps
         else:
             raise TypeError(
                 f"Wrong type for fps: {type(fps)} - expected float, Fraction, or FrameTimecode"
@@ -214,9 +217,11 @@ def __init__(
         if isinstance(timecode, Timecode):
             self._timecode = timecode
             return
+
         # Process the timecode value, storing it as an exact number of frames only if required.
         if isinstance(timecode, str) and timecode.isdigit():
             timecode = int(timecode)
+
         if isinstance(timecode, str):
             self._seconds = self._timecode_to_seconds(timecode)
         elif isinstance(timecode, float):
@@ -232,6 +237,8 @@ def __init__(
 
     @property
     def frame_num(self) -> ty.Optional[int]:
+        """The frame number. This value will be an estimate if the video is VFR. Prefer using the
+        `pts` property."""
         if self._timecode:
             # We need to audit anything currently using this property to guarantee temporal
             # consistency when handling VFR videos (i.e. no assumptions on fixed frame rate).
@@ -249,8 +256,24 @@ def frame_num(self) -> ty.Optional[int]:
         return self._frame_num
 
     @property
-    def framerate(self) -> ty.Optional[float]:
-        return float(self._framerate)
+    def framerate(self) -> float:
+        """The framerate to use for distance between frames and to calculate frame numbers.
+        For a VFR video, this may just be the average framerate."""
+        return float(self._rate)
+
+    @property
+    def time_base(self) -> Fraction:
+        """The time base in which presentation time is calculated."""
+        if self._timecode:
+            return self._timecode.time_base
+        return 1 / self._rate
+
+    @property
+    def pts(self) -> int:
+        """The presentation timestamp of the frame in units of `time_base`."""
+        if self._timecode:
+            return self._timecode.pts
+        return self.frame_num
 
     def get_frames(self) -> int:
         """[DEPRECATED] Get the current time/position in number of frames.
@@ -302,8 +325,7 @@ def seconds(self) -> float:
             return self._timecode.seconds
         if self._seconds:
             return self._seconds
-        # Assume constant framerate if we don't have timing information.
-        return float(self._frame_num) / self._framerate
+        return float(self._frame_num / self._rate)
 
     def get_seconds(self) -> float:
         """[DEPRECATED] Get the frame's position in number of seconds.
@@ -372,7 +394,7 @@ def _seconds_to_frames(self, seconds: float) -> int:
 
         *NOTE*: This will not be correct for variable framerate videos.
         """
-        return round(seconds * self._framerate)
+        return round(seconds * self._rate)
 
     def _parse_timecode_number(self, timecode: ty.Union[int, float]) -> int:
         """Parse a timecode number, storing it as the exact number of frames.
@@ -406,7 +428,7 @@ def _timecode_to_seconds(self, input: str) -> float:
         Raises:
             ValueError: Value could not be parsed correctly.
         """
-        assert self._framerate is not None and self._framerate > MAX_FPS_DELTA
+        assert self._rate is not None and self._rate > MAX_FPS_DELTA
         input = input.strip()
         # Exact number of frames N
         if input.isdigit():
@@ -452,7 +474,7 @@ def _get_other_as_frames(self, other: ty.Union[int, float, str, "FrameTimecode"]
             return self._seconds_to_frames(self._timecode_to_seconds(other))
         if isinstance(other, FrameTimecode):
             # If comparing two FrameTimecodes, they must have the same framerate for frame-based operations.
-            if self._framerate and other._framerate and not self.equal_framerate(other._framerate):
+            if self._rate and other._rate and not self.equal_framerate(other._rate):
                 raise ValueError(
                     "FrameTimecode instances require equal framerate for frame-based arithmetic."
                 )
@@ -530,7 +552,7 @@ def __iadd__(self, other: ty.Union[int, float, str, "FrameTimecode"]) -> "FrameT
                 time_base=timecode.time_base,
             )
             self._seconds = None
-            self._framerate = None
+            self._rate = None
             self._frame_num = None
             return self
 
@@ -573,7 +595,7 @@ def __isub__(self, other: ty.Union[int, float, str, "FrameTimecode"]) -> "FrameT
                 time_base=timecode.time_base,
             )
             self._seconds = None
-            self._framerate = None
+            self._rate = None
             self._frame_num = None
             return self
 
@@ -610,8 +632,8 @@ def __repr__(self) -> str:
         if self._timecode:
             return f"{self.get_timecode()} [pts={self._timecode.pts}, time_base={self._timecode.time_base}]"
         if self._seconds is not None:
-            return f"{self.get_timecode()} [seconds={self._seconds}, fps={self._framerate}]"
-        return f"{self.get_timecode()} [frame_num={self._frame_num}, fps={self._framerate}]"
+            return f"{self.get_timecode()} [seconds={self._seconds}, fps={self._rate}]"
+        return f"{self.get_timecode()} [frame_num={self._frame_num}, fps={self._rate}]"
 
     def __hash__(self) -> int:
         if self._timecode:
@@ -628,7 +650,7 @@ def _get_other_as_seconds(self, other: ty.Union[int, float, str, "FrameTimecode"
                 if _USE_PTS_IN_DEVELOPMENT and other == 1:
                     return self.seconds
                 raise NotImplementedError()
-            return float(other) / self._framerate
+            return float(other) / self._rate
         if isinstance(other, float):
             return other
         if isinstance(other, str):
@@ -639,4 +661,4 @@ def _get_other_as_seconds(self, other: ty.Union[int, float, str, "FrameTimecode"
 
 
 def _compare_as_fixed(a: FrameTimecode, b: ty.Any) -> bool:
-    return a._framerate is not None and isinstance(b, FrameTimecode) and b._framerate is not None
+    return a._rate is not None and isinstance(b, FrameTimecode) and b._rate is not None

scenedetect/detector.py

@@ -30,7 +30,7 @@
 
 import numpy
 
-from scenedetect.common import _USE_PTS_IN_DEVELOPMENT, FrameTimecode
+from scenedetect.common import FrameTimecode
 from scenedetect.stats_manager import StatsManager
 
 
@@ -49,7 +49,7 @@ def __init__(self):
     def process_frame(
         self, timecode: FrameTimecode, frame_img: numpy.ndarray
     ) -> ty.List[FrameTimecode]:
-        """Process the next frame. `frame_num` is assumed to be sequential.
+        """Process the next frame. `timecode` is assumed to be sequential.
 
         Args:
             timecode: Timecode corresponding to the frame being processed.
@@ -74,9 +74,8 @@ def post_process(self, timecode: int) -> ty.List[FrameTimecode]:
 
     @property
     def event_buffer_length(self) -> int:
-        """The amount of frames a given event can be buffered for, in time. Represents maximum
-        amount any event can be behind `frame_number` in the result of :meth:`process_frame`.
-        """
+        """The amount of frames a given event can be buffered for, in time. This must be set to the
+        amount of frames a detector might emit an event in the past."""
         return 0
 
     # Frame Stats/Metrics
@@ -135,30 +134,32 @@ def max_behind(self) -> int:
     def filter(self, timecode: FrameTimecode, above_threshold: bool) -> ty.List[FrameTimecode]:
         if not self._filter_length > 0:
             return [timecode] if above_threshold else []
-        if _USE_PTS_IN_DEVELOPMENT:
-            raise NotImplementedError("TODO: Change filter to use units of time instead of frames.")
         if self._last_above is None:
             self._last_above = timecode
         if self._mode == FlashFilter.Mode.MERGE:
-            return self._filter_merge(frame_num=timecode, above_threshold=above_threshold)
+            return self._filter_merge(timecode=timecode, above_threshold=above_threshold)
         elif self._mode == FlashFilter.Mode.SUPPRESS:
-            return self._filter_suppress(frame_num=timecode, above_threshold=above_threshold)
+            return self._filter_suppress(timecode=timecode, above_threshold=above_threshold)
         raise RuntimeError("Unhandled FlashFilter mode.")
 
-    def _filter_suppress(self, frame_num: int, above_threshold: bool) -> ty.List[int]:
-        min_length_met: bool = (frame_num - self._last_above) >= self._filter_length
+    def _filter_suppress(self, timecode: FrameTimecode, above_threshold: bool) -> ty.List[int]:
+        framerate = timecode.framerate
+        assert framerate >= 0
+        min_length_met: bool = (timecode - self._last_above) >= (self._filter_length / framerate)
         if not (above_threshold and min_length_met):
             return []
         # Both length and threshold requirements were satisfied. Emit the cut, and wait until both
         # requirements are met again.
-        self._last_above = frame_num
-        return [frame_num]
+        self._last_above = timecode
+        return [timecode]
 
-    def _filter_merge(self, frame_num: int, above_threshold: bool) -> ty.List[int]:
-        min_length_met: bool = (frame_num - self._last_above) >= self._filter_length
+    def _filter_merge(self, timecode: FrameTimecode, above_threshold: bool) -> ty.List[int]:
+        framerate = timecode.framerate
+        assert framerate >= 0
+        min_length_met: bool = (timecode - self._last_above) >= (self._filter_length / framerate)
         # Ensure last frame is always advanced to the most recent one that was above the threshold.
         if above_threshold:
-            self._last_above = frame_num
+            self._last_above = timecode
         if self._merge_triggered:
             # This frame was under the threshold, see if enough frames passed to disable the filter.
             num_merged_frames = self._last_above - self._merge_start
@@ -174,9 +175,9 @@ def _filter_merge(self, frame_num: int, above_threshold: bool) -> ty.List[int]:
         if min_length_met:
             # Only allow the merge filter once the first cut is emitted.
             self._merge_enabled = True
-            return [frame_num]
+            return [timecode]
         # Start merging cuts until the length requirement is met.
         if self._merge_enabled:
             self._merge_triggered = True
-            self._merge_start = frame_num
+            self._merge_start = timecode
         return []

scenedetect/detectors/content_detector.py

@@ -137,7 +137,8 @@ def __init__(
                 raise ValueError("kernel_size must be odd integer >= 3")
             self._kernel = numpy.ones((kernel_size, kernel_size), numpy.uint8)
         self._frame_score: ty.Optional[float] = None
-        # TODO(https://scenedetect.com/issue/168): Handle timecodes in filter.
+        # TODO(https://scenedetect.com/issue/168): Figure out a better long term plan for handling
+        # `min_scene_len` which should be specified in seconds, not frames.
         self._flash_filter = FlashFilter(mode=filter_mode, length=min_scene_len)
 
     def get_metrics(self):

scenedetect/output/image.py

@@ -293,7 +293,8 @@ def image_save_thread(self, save_queue: queue.Queue, progress_bar: tqdm):
     def generate_timecode_list(self, scene_list: SceneList) -> ty.List[ty.Iterable[FrameTimecode]]:
         """Generates a list of timecodes for each scene in `scene_list` based on the current config
         parameters."""
-        framerate = scene_list[0][0]._framerate
+        # TODO(v0.7): This needs to be fixed as part of PTS overhaul.
+        framerate = scene_list[0][0].framerate
         # TODO(v1.0): Split up into multiple sub-expressions so auto-formatter works correctly.
         return [
             (
@@ -450,7 +451,7 @@ def save_images(
     image_num_format = "%0"
     image_num_format += str(math.floor(math.log(num_images, 10)) + 2) + "d"
 
-    framerate = scene_list[0][0]._framerate
+    framerate = scene_list[0][0]._rate
 
     # TODO(v1.0): Split up into multiple sub-expressions so auto-formatter works correctly.
     timecode_list = [