From 3db4d03e2df3857d9278aa0fc090cb0c8cf69d74 Mon Sep 17 00:00:00 2001 From: Caglar Pir Date: Fri, 23 Oct 2026 12:05:16 +0200 Subject: [PATCH 1/5] Read a video's GPS clock as a straight line at the rate it fits Since #888, GPX sync places a track, and sampling stamps frames, through every GPS timestamp of the video. The cameras whose timestamps leave a straight line record in real time, so video time maps to GPS time along a straight line, and their timestamps jitter about it. A GoPro stamps each block of 1.001 seconds of video up to 1.7 seconds off the line through the rest: on a GoPro MAX, the accelerometer timestamps of those blocks advance 0.996 to 1.006 seconds each, while their GPS timestamps advance 0 to 3.8 seconds. A PANOX V2 stamps some fixes up to 6 seconds late. #888 followed that jitter. It placed a GPX up to 1.4 s off the line on a GoPro HERO7, 1.6 to 2.9 s on GoPro MAX videos with poor GPS, and 5.9 s on a PANOX V2. It stamped interval samples 5 s apart from 3.7 to 6.3 s apart (HERO7), and 10 s apart from 7.2 to 12.8 s apart (PANOX V2). Its message put the move of a GPX on a GoPro MAX at up to 56 ms, and that of GoPro interval stamps at up to 1.5 s; on these videos, both moved by up to 2.9 s. The LabPano Pilot One and Era, which #888 was written for, need no more than a straight clock. Their timestamps fit one at 1.143 GPS seconds per second of video, within 0.3 s over 14 minutes (0.7 s on the Era). The clock was read at 1x only because any rate below the timelapse rate of 1.5 was rounded to 1. Read the clock as the least-squares line through the timestamps, at the rate it fits, anchored at their median as before, and use it everywhere: GPX sync places the track by it, interval samples and distance samples of the video's own track are stamped by it, and the geotagging stage times the video's track at its rate. The fit leaves out the timestamps that #888 showed to be wrong: those out of order, those before a jump of more than a day, and stale first ones. The rate is 1x when it is within 5% of 1x, or when it departs from 1x by less than 2 seconds over the timestamps. On sample videos, cameras that record in real time measure within 0.4% of 1x over 20 seconds or more, or 1.6% for a GoPro MAX whose GPS glitches. Clips of a few seconds measure up to 14% off 1x, a drift of at most 0.3 s. The LabPanos measure 1.093 to 1.143, a drift of at least 15 s. A rate above 1.5 must still show in the median step between timestamps, so one timestamp far off makes no timelapse. Timelapses get a straight clock too. Once their stale first timestamps are left out, those of GoPro MAX, MAX 2 and Rove timelapses lie within 0.33 GPS seconds of the line, but for one GoPro MAX timelapse within 0.82 (55 ms of video), less than the jitter of a normal video. On sample videos from 9 cameras: - Without a GPX, process writes the same tracks for all of them. - A GPX made from a video's own track moves onto the straight line: by up to 1.4 s on the HERO7, 2.9 s and 1.6 s on GoPro MAX videos with poor GPS, 0.65 s on a GoPro MAX 2, 5.9 s on a PANOX V2, and up to 0.06 s on other GoPros. Interval samples are stamped evenly, 5.000 s or 10.000 s apart, and those of a LabPano 11.426 s apart for 10 s of video. Distance samples of the video's own track are stamped by the clock, and their positions do not change. - A GPX made from a LabPano's own track lands within 0.26 s of its timestamps over 14 minutes, and within 0.61 s on the Era. GPX distance samples of the 2 fps Era are no longer dropped: #888 timed its frames alternately fast and slow, so half of them fell under the sample distance. One LabPano video is the exception: its GPS drops out for 6 seconds of video, its positions jump 64 m, and its GPS time advances 2 s over 8 seconds of video. A GPX of its own track is placed up to 3.8 s off its timestamps, and interval samples geotagged from it up to 65 m off its own positions. - GoPro MAX timelapses keep #852's results: their placements move by up to 0.05 s, and all their distance samples are described. - A GPX that ends exactly at a video's own first or last timestamp, as one made from its track does, can now miss a sample at the very start or end of the video, which is stamped just past it and reported as outside the track: 1 to 4 samples on 4 of these videos. A logger track that covers the video does not. - BlackVue, Viofo and Ricoh outputs do not change, but for 1 ms rounding of BlackVue GPX track times. --- .../geotag/geotag_images_from_video.py | 16 +- .../geotag/video_extractors/gpx.py | 292 +++++++----------- mapillary_tools/sample_video.py | 78 ++--- tests/integration/test_gopro.py | 21 +- tests/integration/test_process.py | 21 +- tests/integration/test_upload.py | 9 +- tests/unit/test_geotag_images_from_video.py | 38 +-- tests/unit/test_gpx_video_clip.py | 19 +- tests/unit/test_gpx_video_clock.py | 10 +- tests/unit/test_gpx_video_sync.py | 28 +- ...meline.py => test_gpx_video_timestamps.py} | 213 +++++++------ tests/unit/test_sample_video.py | 72 +++-- 12 files changed, 367 insertions(+), 450 deletions(-) rename tests/unit/{test_gps_timeline.py => test_gpx_video_timestamps.py} (50%) diff --git a/mapillary_tools/geotag/geotag_images_from_video.py b/mapillary_tools/geotag/geotag_images_from_video.py index 092fe2f42..6422a5c99 100644 --- a/mapillary_tools/geotag/geotag_images_from_video.py +++ b/mapillary_tools/geotag/geotag_images_from_video.py @@ -20,7 +20,7 @@ from .base import GeotagImagesFromGeneric from .geotag_images_from_gpx import GeotagImagesFromGPX from .geotag_videos_from_video import GeotagVideosFromVideo -from .video_extractors.gpx import GPSTimeline, GPXVideoExtractor +from .video_extractors.gpx import GPXVideoExtractor LOG = logging.getLogger(__name__) @@ -109,21 +109,15 @@ def _on_capture_timeline( ) -> T.Sequence[geo.Point]: """ Time the video's GPS points in GPS seconds from the start of the video, - like the samples, which are stamped with their capture times, as its - GPS timestamps tell: a timelapse packs several into each second of + like the samples, which are stamped with their capture times, at the + pace of its GPS clock: a timelapse packs several into each second of video, and some clocks gain on the video. """ clock = GPXVideoExtractor._read_video_clock(points) - if clock is None: + if clock is None or clock.rate == 1.0: return points - timeline = GPSTimeline.read(points, clock.rate) - assert timeline is not None, "a GPS clock is read from GPS timestamps" - start = timeline.to_unix_time(0.0) - return [ - dataclasses.replace(p, time=timeline.to_unix_time(p.time) - start) - for p in points - ] + return [dataclasses.replace(p, time=p.time * clock.rate) for p in points] class GeotagImageSamplesFromVideo(GeotagImagesFromGeneric): diff --git a/mapillary_tools/geotag/video_extractors/gpx.py b/mapillary_tools/geotag/video_extractors/gpx.py index 02a0ebd1f..3144a6824 100644 --- a/mapillary_tools/geotag/video_extractors/gpx.py +++ b/mapillary_tools/geotag/video_extractors/gpx.py @@ -46,9 +46,10 @@ # a GPS logger left running after the video, but a log of several trips _MAX_RECORDING_SECONDS = 24 * 3600 -# A GPS clock that runs this much faster than the video clock is a timelapse. -# Normal videos stay well below it, although over a few seconds of GPS -# timestamped once per block the estimate can be 20% off +# A GPS clock that runs this much faster than the video clock is a timelapse, +# and the steps between its timestamps show it too. One timestamp far off the +# rest, or one step of the clock, can pull the least-squares slope of a normal +# video past it _MIN_TIMELAPSE_RATE = 1.5 # No GPS fix can be timestamped before the GPS epoch, 1980-01-06. GPX files @@ -70,6 +71,18 @@ # in steps of a second per half second, 2 minutes over a 14-minute video _STALE_STEP_RATE = 1.5 +# A GPS clock that runs within this fraction of the video's pace runs at it. +# Over 20 seconds or more, cameras that record in real time measure within +# 0.4% of 1x, or 1.6% for a GoPro MAX whose GPS glitches, while the clock of a +# LabPano runs 9% to 14% faster than its video +_MAX_CLOCK_RATE_ERROR = 0.05 + +# Nor can GPS timestamps measure a clock that departs from the video's pace by +# less than their own jitter over the video time they span: GoPro stamps blocks +# of points up to 1.7 seconds off the line through the rest, and clips a few +# seconds long measure up to 15% off 1x, if by no more than 0.3 seconds +_MAX_CLOCK_DRIFT_SECONDS = 2.0 + class SyncMode(enum.Enum): # Sync by video GPS timestamps if found, otherwise rebase @@ -83,7 +96,10 @@ class SyncMode(enum.Enum): @dataclasses.dataclass(frozen=True) class VideoClock: """ - Maps between the video time and the Unix time of the GPS clock in the video + Maps between the video time and the Unix time of the GPS clock in the video: + a straight line, as a camera records its frames at a steady pace, whether + in real time or as a timelapse. The GPS timestamps of the video jitter + about it >>> clock = VideoClock(video_time=2.0, unix_time=1000.0) >>> clock.to_video_time(1030.0) @@ -107,124 +123,6 @@ def to_unix_time(self, video_time: float) -> float: return self.unix_time + (video_time - self.video_time) * self.rate -@dataclasses.dataclass(frozen=True) -class GPSTimeline: - """ - The Unix time of the video's GPS clock along the video: interpolated - between its GPS timestamps, and extrapolated beyond them at the clock's - rate. A straight clock does not follow them: the clock of a timelapse - drifts from its rate over the video, the first timestamp of a GoPro MAX - timelapse can be seconds early, and some cameras gain on the video in - steps. - - >>> timeline = GPSTimeline((0.0, 1.0, 2.0), (100.0, 104.0, 114.0), rate=10.0) - >>> timeline.to_unix_time(0.5) - 102.0 - >>> timeline.to_unix_time(3.0) - 124.0 - >>> timeline.to_video_time(109.0) - 1.5 - >>> timeline.to_video_time(95.0) - -0.5 - >>> timeline.starting_at(1.0, 200.0).to_unix_time(2.0) - 210.0 - """ - - video_times: tuple[float, ...] - unix_times: tuple[float, ...] - # GPS seconds per video second beyond the timestamps - rate: float - - @classmethod - def read(cls, points: T.Sequence[geo.Point], rate: float) -> GPSTimeline | None: - """ - Read the timeline from the GPS timestamps of the points, or None if they - have none. Leave out the timestamps that the others show to be wrong: - those out of order, those before a jump of days (from a placeholder - date to the GPS time), and stale first ones. - """ - pairs: list[tuple[float, float]] = [] - for p in points: - unix_time = p.get_unix_time() - if unix_time is None: - continue - if pairs and unix_time == pairs[-1][1]: - # GoPro GPS5 stamps a whole block (~1s) of points with one - # timestamp. Keep the first point of the block - continue - if pairs and p.time <= pairs[-1][0]: - # Two timestamps at one video time - continue - pairs.append((p.time, unix_time)) - - # A timestamp out of order with the others is wrong, and could not be - # interpolated back to the video - pairs = _increasing_subsequence(pairs) - if not pairs: - return None - - # GoPro stamps a placeholder date until the GPS acquires the time - start = 0 - for idx in range(1, len(pairs)): - (v0, u0), (v1, u1) = pairs[idx - 1], pairs[idx] - if _IMPLAUSIBLE_OFFSET_SECONDS < (u1 - u0) - (v1 - v0) * rate: - start = idx - - # Stale first timestamps (see _STALE_STEP_RATE). Judged by the steps of - # the rest, not by the rate: that is 1 for timestamps over less than a - # second of video, as those of a short timelapse, too few to measure it - pace = GPXVideoExtractor._median_step_rate(pairs[start + 1 :]) - while pace is not None and start + 1 < len(pairs): - (v0, u0), (v1, u1) = pairs[start], pairs[start + 1] - if (u1 - u0) <= (v1 - v0) * pace * _STALE_STEP_RATE: - break - start += 1 - - return cls( - video_times=tuple(v for v, _ in pairs[start:]), - unix_times=tuple(u for _, u in pairs[start:]), - rate=rate, - ) - - def to_unix_time(self, video_time: float) -> float: - return _interpolate_line( - self.video_times, self.unix_times, video_time, self.rate - ) - - def to_video_time(self, unix_time: float) -> float: - return _interpolate_line( - self.unix_times, self.video_times, unix_time, 1 / self.rate - ) - - def starting_at(self, video_time: float, unix_time: float) -> GPSTimeline: - """ - The timeline shifted to be at **unix_time** at **video_time**: at the - pace of the GPS clock, from a start that the clock does not tell - """ - shift = unix_time - self.to_unix_time(video_time) - return dataclasses.replace( - self, unix_times=tuple(u + shift for u in self.unix_times) - ) - - -def _interpolate_line( - xs: T.Sequence[float], ys: T.Sequence[float], x: float, slope: float -) -> float: - """ - Interpolate y at x along the line through the points (xs, ys), xs in - increasing order, and extrapolate it beyond them at **slope** - """ - idx = bisect.bisect_right(xs, x) - if 0 < idx < len(xs): - x0, x1 = xs[idx - 1], xs[idx] - y0, y1 = ys[idx - 1], ys[idx] - return y0 + (y1 - y0) * (x - x0) / (x1 - x0) - - # Before the first point or after the last - anchor = 0 if idx == 0 else -1 - return ys[anchor] + (x - xs[anchor]) * slope - - def _increasing_subsequence( pairs: T.Sequence[tuple[float, float]], ) -> list[tuple[float, float]]: @@ -315,9 +213,7 @@ def extract(self) -> types.VideoMetadata: ) video_clock = self._read_video_clock(native_video_metadata.points) - clock: VideoClock | GPSTimeline | None = self._sync_clock( - video_clock, gpx_points, noisy - ) + clock = self._sync_clock(video_clock, gpx_points, noisy) if video_clock is not None and clock is None: LOG.warning( "The GPS clock of %s, whose GPS was rejected as noise, is %.1f days away from %s. " @@ -327,17 +223,10 @@ def extract(self) -> types.VideoMetadata: self.gpx_path, self.file_format, ) - if clock is not None and not noisy: - # Through the GPS timestamps of the video, which no straight clock - # follows. GPS rejected as noise gives the clock only - timeline = GPSTimeline.read(native_video_metadata.points, clock.rate) - assert timeline is not None, "a GPS clock is read from GPS timestamps" - clock = timeline if clock is not None and gpx_points and self.sync_mode is SyncMode.REBASE: - # At the pace of the GPS clock from the start of the track, which - # distance sampling, reading the frames by their composition times, - # puts at that of the first frame - clock = self._start_with_the_video(clock, gpx_points[0].time) + # Distance sampling reads the frames by their composition times: + # start the track at that of the first frame + clock = dataclasses.replace(clock, video_time=self._gpx_start_video_time()) if clock is None: # Its GPS has no clock to sync with, as if it had no GPS @@ -359,7 +248,7 @@ def extract(self) -> types.VideoMetadata: ) if clock.rate != 1.0: LOG.info( - "The GPS clock in %s runs %.1f times faster than the video (timelapse). Syncing %s accordingly", + "The GPS clock in %s runs %.3g times faster than the video. Syncing %s accordingly", self.video_path, clock.rate, self.gpx_path, @@ -424,7 +313,7 @@ def _parse_tracks(self) -> list[list[geo.Point]]: ) from ex def _clip_to_video( - self, gpx_points: list[geo.Point], clock: VideoClock | GPSTimeline | None + self, gpx_points: list[geo.Point], clock: VideoClock | None ) -> list[geo.Point]: """ Clip a GPX track to the video, from video time 0 to where the video @@ -486,20 +375,6 @@ def _place_without_video_gps( self._rebase_times(gpx_points, rate=ratio) return None - def _start_with_the_video( - self, clock: VideoClock | GPSTimeline, gpx_start_time: float - ) -> VideoClock | GPSTimeline: - """ - The **clock** moved to start the track, at **gpx_start_time**, with - the video, as --interpolation_use_gpx_start_time says it does - """ - video_time = self._gpx_start_video_time() - if isinstance(clock, GPSTimeline): - return clock.starting_at(video_time, gpx_start_time) - return VideoClock( - video_time=video_time, unix_time=gpx_start_time, rate=clock.rate - ) - def _gpx_start_video_time(self) -> float: """ The video time that --interpolation_use_gpx_start_time starts the GPX @@ -557,7 +432,7 @@ def _check_coverage( gpx_points: T.Sequence[geo.Point], gpx_unix_times: tuple[float, float] | None, video_gps_points: T.Sequence[geo.Point], - clock: VideoClock | GPSTimeline | None, + clock: VideoClock | None, rate: float, ) -> None: """ @@ -969,7 +844,12 @@ def _read_video_clock( cls, video_gps_points: T.Sequence[geo.Point] ) -> VideoClock | None: """ - Read the GPS clock from the video GPS points, or None if they have no timestamps + Read the GPS clock from the video GPS points, or None if they have no + timestamps: the straight line that their timestamps fit, at 1x unless + they measure a pace off it. The fit assumes that the clock runs at a + steady pace, as the camera records its frames: a step of the clock, or + a GPS dropout that it does not recover from, leaves the timestamps off + any line >>> GPXVideoExtractor._read_video_clock([]) is None True @@ -977,9 +857,54 @@ def _read_video_clock( >>> GPXVideoExtractor._read_video_clock([point]) is None True """ - # (video time, Unix time) pairs - pairs: list[tuple[float, float]] = [] + pairs = cls._read_timestamps(video_gps_points) + if not pairs: + return None + + rate = cls._estimate_clock_rate(pairs) + if rate is None or abs(rate - 1.0) <= max( + _MAX_CLOCK_RATE_ERROR, + _MAX_CLOCK_DRIFT_SECONDS / (pairs[-1][0] - pairs[0][0]), + ): + # See _MAX_CLOCK_RATE_ERROR and _MAX_CLOCK_DRIFT_SECONDS + rate = 1.0 + elif _MIN_TIMELAPSE_RATE < rate: + # A timestamp far off the rest, or a step of the clock, can pull the + # least-squares slope far off, so a timelapse must show in the + # median step between neighbouring pairs too. That is too coarse to + # measure the rate, but not to tell a timelapse from a normal video + step_rate = cls._median_step_rate(pairs) + if step_rate is None or step_rate <= _MIN_TIMELAPSE_RATE: + rate = 1.0 + + # Anchored at its own video time, which is not 0 when the video's GPS + # starts late (e.g. GoPro drops noisy fixes at the start of a recording), + # and at the median of all its timestamps mapped back to that time, not + # at the first alone: GPS clocks jitter by a second or so against the + # video, and a PANOX V2 stamps some fixes seconds late + video_time = pairs[0][0] + unix_time = statistics.median(u - (v - video_time) * rate for v, u in pairs) + return VideoClock(video_time=video_time, unix_time=unix_time, rate=rate) + + @classmethod + def _read_timestamps( + cls, video_gps_points: T.Sequence[geo.Point] + ) -> list[tuple[float, float]]: + """ + Read the (video time, Unix time) pairs of the GPS timestamps of the + points, leaving out those that the others show to be wrong: those out + of order, those before a jump of days (from a placeholder date to the + GPS time), and stale first ones + + >>> points = [ + ... telemetry.GPSPoint(time=v, lat=1, lon=1, alt=None, angle=None, epoch_time=u, fix=None, precision=None, ground_speed=None) + ... for v, u in [(0, 992.4), (0.4, 1000.4), (1.4, 1001.4), (1.8, 1001.4), (2.4, 1002.4)] + ... ] + >>> GPXVideoExtractor._read_timestamps(points) + [(0.4, 1000.4), (1.4, 1001.4), (2.4, 1002.4)] + """ + pairs: list[tuple[float, float]] = [] for p in video_gps_points: # Video GPS timestamps are stored in whatever epoch their container # uses (CAMM records GPS time, GoPro records Unix time), so go @@ -991,51 +916,50 @@ def _read_video_clock( # GoPro GPS5 stamps a whole block (~1s) of points with one # timestamp. Keep the first point of the block continue + if pairs and p.time <= pairs[-1][0]: + # Two timestamps at one video time + continue pairs.append((p.time, unix_time)) - if not pairs: - return None + # A timestamp out of order with the others is wrong + pairs = _increasing_subsequence(pairs) - rate = cls._estimate_clock_rate(pairs) - # One timestamp far off (e.g. a placeholder date before GPS time is - # acquired) can pull the least-squares slope far off, so a timelapse - # must show in the median step between neighbouring pairs too. That is - # too coarse to measure the rate, but not to tell a timelapse from a - # normal video - step_rate = cls._median_step_rate(pairs) - if not ( - rate is not None - and _MIN_TIMELAPSE_RATE < rate - and step_rate is not None - and _MIN_TIMELAPSE_RATE < step_rate - ): - rate = 1.0 + pace = cls._median_step_rate(pairs) + if pace is None: + return pairs - # Anchored at its own video time, which is not 0 when the video's GPS - # starts late (e.g. GoPro drops noisy fixes at the start of a recording), - # and at the median of all its timestamps mapped back to that time, not - # at the first alone: that can be seconds off the rest (the first GPS - # block of a GoPro timelapse, a stale first fix), and some GPS clocks - # wander by a second or so against the video - video_time = pairs[0][0] - unix_time = statistics.median(u - (v - video_time) * rate for v, u in pairs) + # GoPro stamps a placeholder date until the GPS acquires the time + start = 0 + for idx in range(1, len(pairs)): + (v0, u0), (v1, u1) = pairs[idx - 1], pairs[idx] + if _IMPLAUSIBLE_OFFSET_SECONDS < (u1 - u0) - (v1 - v0) * pace: + start = idx - return VideoClock(video_time=video_time, unix_time=unix_time, rate=rate) + # Stale first timestamps (see _STALE_STEP_RATE). Judged by the steps of + # the rest: the rate is fitted to the timestamps that are left + pace = cls._median_step_rate(pairs[start + 1 :]) + while pace is not None and start + 1 < len(pairs): + (v0, u0), (v1, u1) = pairs[start], pairs[start + 1] + if (u1 - u0) <= (v1 - v0) * pace * _STALE_STEP_RATE: + break + start += 1 + + return pairs[start:] @classmethod def _estimate_clock_rate( cls, pairs: T.Sequence[tuple[float, float]] ) -> float | None: """ - Least-squares slope of the Unix time over the video time, or None if the - pairs span less than a second of video + Least-squares slope of the Unix time over the video time, or None + without two pairs at different video times >>> GPXVideoExtractor._estimate_clock_rate([(0, 100), (1, 115), (2, 130)]) 15.0 - >>> GPXVideoExtractor._estimate_clock_rate([(0, 100), (0.5, 107.5)]) is None + >>> GPXVideoExtractor._estimate_clock_rate([(0, 100)]) is None True """ - if len(pairs) < 2 or pairs[-1][0] - pairs[0][0] < 1.0: + if len(pairs) < 2 or pairs[-1][0] <= pairs[0][0]: return None n = len(pairs) diff --git a/mapillary_tools/sample_video.py b/mapillary_tools/sample_video.py index 2ebcf24ce..17a47840c 100644 --- a/mapillary_tools/sample_video.py +++ b/mapillary_tools/sample_video.py @@ -36,7 +36,7 @@ SourceType, VideoSyncOption, ) -from .geotag.video_extractors.gpx import GPSTimeline, GPXVideoExtractor, VideoClock +from .geotag.video_extractors.gpx import GPXVideoExtractor, VideoClock from .geotag.video_extractors.native import NativeVideoExtractor from .geotag.video_extractors.nmea import NMEAVideoExtractor from .mp4 import mp4_sample_parser, simple_mp4_parser as sparser @@ -473,12 +473,12 @@ def _read_telemetry(video_path: Path) -> T.Sequence[geo.Point]: return [] -def _telemetry_clock_rate( +def _telemetry_clock( video_path: Path, telemetry_points: T.Sequence[geo.Point] -) -> float | None: +) -> VideoClock | None: """ - GPS seconds per video second of the video's own GPS clock: more than 1 if - the video is a timelapse, or None if it has no GPS clock + The video's own GPS clock, or None if it has none. It runs more than 1 GPS + second per video second if the video is a timelapse """ clock = GPXVideoExtractor._read_video_clock(telemetry_points) if clock is None: @@ -486,11 +486,11 @@ def _telemetry_clock_rate( if clock.rate != 1.0: LOG.info( - "The GPS clock in %s runs %.1f times faster than the video (timelapse)", + "The GPS clock in %s runs %.3g times faster than the video", video_path.name, clock.rate, ) - return clock.rate + return clock def _parse_filename_time(video_path: Path) -> datetime.datetime | None: @@ -770,48 +770,36 @@ def _sample_single_video_by_interval( telemetry_points = _read_telemetry(video_path) # Frames are stamped with their capture times, which a timelapse packs - # several seconds of into each second of video: as the video's own GPS - # timestamps tell, and for a video without a GPS clock, at - # --video_duration_ratio - clock_rate = _telemetry_clock_rate(video_path, telemetry_points) + # several seconds of into each second of video: by the video's own GPS + # clock, as GPX sync places a GPX and distance samples are stamped, and for + # a video without a GPS clock, at --video_duration_ratio + clock = _telemetry_clock(video_path, telemetry_points) # GPS rejected as noise gives no telemetry, but GPX sync reads its clock, # and the frames are stamped by it, where GPX sync places them along the # GPX, so that the geotagging stage finds them there - gpx_clock: VideoClock | None = None - if clock_rate is None: - gpx_clock = _gpx_sync_clock(video_path, geotag_options) - if gpx_clock is not None: - clock_rate = gpx_clock.rate - timeline: GPSTimeline | None = None - if clock_rate is None: + if clock is None: + clock = _gpx_sync_clock(video_path, geotag_options) + if clock is None: rate = duration_ratio else: - rate = clock_rate + rate = clock.rate if duration_ratio != 1.0: LOG.warning( "Ignoring --video_duration_ratio %g for %s, whose own GPS clock runs at %.3gx", duration_ratio, video_path.name, - clock_rate, + clock.rate, ) - if gpx_clock is None: - # As distance samples are stamped, and as the geotagging stage - # times the video's track to place the samples along it - timeline = GPSTimeline.read(telemetry_points, clock_rate) - if clock_rate is None and video_sync is not None: + if clock is None and video_sync is not None: _check_gpx_start_by_interval(video_path, geotag_options, video_sync) probe = _probe_video(ffmpeg, video_path) if start_time is None: - if timeline is not None: + if clock is not None: start_time = datetime.datetime.fromtimestamp( - timeline.to_unix_time(0.0), tz=datetime.timezone.utc - ) - elif gpx_clock is not None: - start_time = datetime.datetime.fromtimestamp( - gpx_clock.to_unix_time(0.0), tz=datetime.timezone.utc + clock.to_unix_time(0.0), tz=datetime.timezone.utc ) else: start_time = _extract_video_start_time(video_path, probe, telemetry_points) @@ -843,11 +831,7 @@ def _sample_single_video_by_interval( continue # extract_frames() produces 1-based frame indices so we need to subtract 1 here video_time = (frame_idx_1based - 1) * sample_interval - if timeline is None: - seconds = video_time * rate - else: - seconds = timeline.to_unix_time(video_time) - timeline.to_unix_time(0.0) - timestamp = start_time + datetime.timedelta(seconds=seconds) + timestamp = start_time + datetime.timedelta(seconds=video_time * rate) exif_edit = ExifEdit(sample_paths[0]) exif_edit.add_date_time_original(timestamp) exif_edit.add_gps_datetime(timestamp) @@ -1057,11 +1041,11 @@ def _sample_single_video_by_distance( last_unix_time = max(track_unix_times, default=None) # Read from the video itself, a track with a GPS clock of its own paces the - # video, and stamps the frames along its timeline, as interval samples are. - # A GPX source warns of the options that its sync ignores itself, and its - # sync placed the track along the video's clock already: its timestamps - # stamp the frames as they are - track_timeline: GPSTimeline | None = None + # video, and its clock stamps the frames, as interval samples are. A GPX + # source warns of the options that its sync ignores itself, and its sync + # placed the track along the video's clock already: its timestamps stamp + # the frames as they are + track_clock: VideoClock | None = None if not any( option.source in factory.EXTERNAL_GPS_SOURCES for option in geotag_options ): @@ -1073,14 +1057,12 @@ def _sample_single_video_by_distance( start_time, ) track_clock = GPXVideoExtractor._read_video_clock(video_metadata.points) - if track_clock is not None: - track_timeline = GPSTimeline.read(video_metadata.points, track_clock.rate) - # Frames along the timeline, or at points with an absolute timestamp, are - # timestamped from it below, so only the others need the start time + # Frames stamped by the clock, or at points with an absolute timestamp, + # are timestamped below, so only the others need the start time if ( start_time is None - and track_timeline is None + and track_clock is None and any( interp.get_unix_time() is None for _, interp in sample_points_by_frame_idx.values() @@ -1130,8 +1112,8 @@ def _sample_single_video_by_distance( gps_unix_time = ( interp.get_unix_time() - if track_timeline is None - else track_timeline.to_unix_time(interp.time) + if track_clock is None + else track_clock.to_unix_time(interp.time) ) if gps_unix_time is not None: assert first_unix_time is not None and last_unix_time is not None diff --git a/tests/integration/test_gopro.py b/tests/integration/test_gopro.py index 115d79507..e7589e452 100644 --- a/tests/integration/test_gopro.py +++ b/tests/integration/test_gopro.py @@ -31,16 +31,17 @@ "MAPILLARY_TOOLS_GOPRO_GPS_PRECISION": "10000000", "MAPILLARY_TOOLS_MAX_CAPTURE_SPEED_KMH": "2000000", # km/h } -# The capture times come from the GPMF GPS timestamps (which put the start at -# 2019-11-18T23:42:10.009Z, without the first, 1.35 seconds stale), not from the -# container's creation time. This camera writes the creation time in local time, -# so reading the start time from there would timestamp every sample 8 hours off. +# The capture times come from the GPMF GPS clock (which puts the start at +# 2019-11-18T23:42:10.003Z, without the first timestamp, 1.36 seconds stale), +# not from the container's creation time. This camera writes the creation time +# in local time, so reading the start time from there would timestamp every +# sample 8 hours off. EXPECTED_DESCS: T.List[T.Any] = [ { "filename": "hero8.mp4/hero8_v_000001.jpg", "filetype": "image", "MAPAltitude": 9540.24, - "MAPCaptureTime": "2019_11_18_23_42_10_009", + "MAPCaptureTime": "2019_11_18_23_42_10_003", "MAPCompassHeading": { "TrueHeading": 123.93587938690177, "MagneticHeading": 123.93587938690177, @@ -55,7 +56,7 @@ "filename": "hero8.mp4/hero8_v_000002.jpg", "filetype": "image", "MAPAltitude": 7112.573717404068, - "MAPCaptureTime": "2019_11_18_23_42_12_052", + "MAPCaptureTime": "2019_11_18_23_42_12_003", "MAPCompassHeading": { "TrueHeading": 140.8665026186285, "MagneticHeading": 140.8665026186285, @@ -70,7 +71,7 @@ "filename": "hero8.mp4/hero8_v_000003.jpg", "filetype": "image", "MAPAltitude": 7463.642846094319, - "MAPCaptureTime": "2019_11_18_23_42_13_976", + "MAPCaptureTime": "2019_11_18_23_42_14_003", "MAPCompassHeading": { "TrueHeading": 138.44255851085705, "MagneticHeading": 138.44255851085705, @@ -85,7 +86,7 @@ "filename": "hero8.mp4/hero8_v_000004.jpg", "filetype": "image", "MAPAltitude": 6909.8168472111465, - "MAPCaptureTime": "2019_11_18_23_42_16_009", + "MAPCaptureTime": "2019_11_18_23_42_16_003", "MAPCompassHeading": { "TrueHeading": 142.23462669862568, "MagneticHeading": 142.23462669862568, @@ -100,7 +101,7 @@ "filename": "hero8.mp4/hero8_v_000005.jpg", "filetype": "image", "MAPAltitude": 7212.594480737465, - "MAPCaptureTime": "2019_11_18_23_42_17_992", + "MAPCaptureTime": "2019_11_18_23_42_18_003", "MAPCompassHeading": { "TrueHeading": 164.70819093235514, "MagneticHeading": 164.70819093235514, @@ -115,7 +116,7 @@ "filename": "hero8.mp4/hero8_v_000006.jpg", "filetype": "image", "MAPAltitude": 7274.361994963208, - "MAPCaptureTime": "2019_11_18_23_42_20_019", + "MAPCaptureTime": "2019_11_18_23_42_20_003", "MAPCompassHeading": { "TrueHeading": 139.71549328876722, "MagneticHeading": 139.71549328876722, diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index a14abe558..46385de91 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -761,7 +761,7 @@ def test_process_video_geotag_source_with_gpx_outside_video( assert descs[0]["error"]["vars"] == { # Its first GPS timestamp is 23:44:42.590, a second before the rest # place the start of the video - "image_time": "2019_11_18_23_44_43_624", + "image_time": "2019_11_18_23_44_43_613", "gpx_start_time": "2025_03_14_07_00_00_000", "gpx_end_time": "2025_03_14_07_01_33_624", } @@ -775,7 +775,7 @@ def test_process_video_from_gpx_with_interpolation_options( images. They were ignored for videos. """ video_path = setup_data.join("gopro_data").join("max-360mode.mp4") - # 3.624 seconds before the GPS timestamps of the video place its start + # 3.613 seconds before the GPS clock of the video starts start = datetime.datetime(2019, 11, 18, 23, 44, 40, tzinfo=datetime.timezone.utc) points = "".join( f'' @@ -809,20 +809,19 @@ def video_time_of_recorded(*options: str) -> float: [point] = [p for p in describe(*options)["MAPGPSTrack"] if p[5] == recorded] return point[0] - # In ms of video, which the GPS timestamps of the video pace at a second - # per second, give or take their jitter of some 40 ms synced = video_time_of_recorded() - assert synced == pytest.approx(10000 - 3624, abs=50) + assert synced == pytest.approx(10000 - 3613, abs=1) - assert video_time_of_recorded("--interpolation_use_gpx_start_time") == ( - pytest.approx(10000, abs=50) + assert video_time_of_recorded("--interpolation_use_gpx_start_time") == 10000 + assert ( + video_time_of_recorded( + "--interpolation_use_gpx_start_time", "--interpolation_offset_time=2.5" + ) + == 7500 ) - assert video_time_of_recorded( - "--interpolation_use_gpx_start_time", "--interpolation_offset_time=2.5" - ) == pytest.approx(7500, abs=50) shifted = video_time_of_recorded("--interpolation_offset_time=2") - assert shifted == pytest.approx(synced - 2000, abs=50) + assert shifted == pytest.approx(synced - 2000, abs=1) # an offset that moves the track off the video fails it, as it does a # track placed by the clock of the video diff --git a/tests/integration/test_upload.py b/tests/integration/test_upload.py index 1afe1bb45..3bfa1e357 100644 --- a/tests/integration/test_upload.py +++ b/tests/integration/test_upload.py @@ -10,7 +10,7 @@ import py.path import pytest -from mapillary_tools.geotag.video_extractors.gpx import GPSTimeline, GPXVideoExtractor +from mapillary_tools.geotag.video_extractors.gpx import GPXVideoExtractor from mapillary_tools.geotag.video_extractors.native import NativeVideoExtractor from .fixtures import ( @@ -52,9 +52,7 @@ def test_upload_video_with_gpx_running_past_both_ends( points = NativeVideoExtractor(Path(video_path)).extract().points clock = GPXVideoExtractor._read_video_clock(points) assert clock is not None - timeline = GPSTimeline.read(points, clock.rate) - assert timeline is not None - video_start = timeline.to_unix_time(0) + video_start = clock.to_unix_time(0) # From 12.5 seconds before the video to about 47 seconds after it trkpts = "".join( f'