From b8156b3453c31411e540e139ce97a0ec2f3a2fac Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:53:06 -0700 Subject: [PATCH 1/5] reset desired speed --- src/components/DriveVideo/index.jsx | 8 ++++++-- src/timeline/playback.js | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/components/DriveVideo/index.jsx b/src/components/DriveVideo/index.jsx index 3b0ea79b..e52b5f83 100644 --- a/src/components/DriveVideo/index.jsx +++ b/src/components/DriveVideo/index.jsx @@ -61,10 +61,13 @@ class DriveVideo extends Component { componentDidMount() { const { dispatch } = this.props; dispatch(resetPlayback()); + setVideoPlayer(this.videoPlayer.current); if (this.videoPlayer.current) { - this.videoPlayer.current.playbackRate = 1; + const internal = this.videoPlayer.current.getInternalPlayer(); + if (internal) { + internal.playbackRate = 1; + } } - setVideoPlayer(this.videoPlayer.current); this.updateVideoSource({}); } @@ -206,6 +209,7 @@ class DriveVideo extends Component { if (src === '' || !prevProps.currentRoute || prevProps.currentRoute?.fullname !== currentRoute.fullname) { src = Video.getQcameraStreamUrl(currentRoute.fullname, currentRoute.share_exp, currentRoute.share_sig); this.setState({ src, videoError: null }); + this.firstSeek = true; } } diff --git a/src/timeline/playback.js b/src/timeline/playback.js index cfe581e0..7d0aea0f 100644 --- a/src/timeline/playback.js +++ b/src/timeline/playback.js @@ -47,6 +47,7 @@ export function reducer(_state, action) { state = { ...state, offset: 0, + desiredPlaySpeed: 1, hasAudio: false, }; break; From 71ea49e84b400d6ebe2d9bbee439d4d71f881b12 Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:53:29 -0700 Subject: [PATCH 2/5] track and cancel raf() loop on exit --- src/components/Timeline/index.jsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/Timeline/index.jsx b/src/components/Timeline/index.jsx index 63aeb9c0..fdee13d6 100644 --- a/src/components/Timeline/index.jsx +++ b/src/components/Timeline/index.jsx @@ -185,7 +185,7 @@ class Timeline extends Component { componentDidMount() { this.mounted = true; - raf(this.getOffset); + this.rafId = raf(this.getOffset); this.componentDidUpdate({}); } @@ -198,6 +198,10 @@ class Timeline extends Component { componentWillUnmount() { this.mounted = false; + if (this.rafId) { + raf.cancel(this.rafId); + this.rafId = null; + } } handleClick(ev) { @@ -284,6 +288,9 @@ class Timeline extends Component { } getOffset() { + if (!this.mounted) { + return; + } let offset; if (this.props.hasAudio && isIos()) { // video with audio doesn't report currentTime properly so we must use onTimeUpdate reported time @@ -300,7 +307,7 @@ class Timeline extends Component { this.rulerRemaining.current.style.left = `${Math.floor(10000 * percent) / 100}%`; this.rulerRemaining.current.style.width = `${100 - Math.floor(10000 * percent) / 100}%`; } - raf(this.getOffset); + this.rafId = raf(this.getOffset); } percentToOffset(perc) { From b6d22929c1afd3430b408325f909ab5f7045062a Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:16:42 -0700 Subject: [PATCH 3/5] better error surfacing and stale audio indicator --- src/components/DriveVideo/index.jsx | 30 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/components/DriveVideo/index.jsx b/src/components/DriveVideo/index.jsx index e52b5f83..32f7f585 100644 --- a/src/components/DriveVideo/index.jsx +++ b/src/components/DriveVideo/index.jsx @@ -104,29 +104,27 @@ class DriveVideo extends Component { dispatch(pause()); } - /** - * @param {Error} e - */ onHlsError(e) { const { dispatch } = this.props; dispatch(bufferVideo(true)); - if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { + if (!e.fatal) return; + else if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { // buffer but no error return; - } - - if (e.type === 'networkError' && (e.response?.code === 404)) { + } else if (e.type === 'networkError' && (e.response?.code === 404)) { this.setState({ videoError: 'This video segment has not uploaded yet or has been deleted.' }); } else { - this.setState({ videoError: e.message }); + const message = + e.reason || + e.response?.text || + e.error?.message || + e.details || + 'Unknown playback error'; + this.setState({ videoError: message }); } } - /** - * @param {Error} e - * @param {any} [data] - */ onVideoError(e, data) { if (!e) { console.warn('Unknown video error', { e, data }); @@ -147,6 +145,7 @@ class DriveVideo extends Component { // autoplay was blocked (e.g. iOS after backgrounding/returning to the app) const { dispatch } = this.props; dispatch(bufferVideo(false)); + dispatch(pause()) return; } @@ -244,10 +243,9 @@ class DriveVideo extends Component { if (isIos()) { // ios does not support hls.js and on other browsers hls.js does not directly play the m3u8 so audioTracks are not visible const videoElement = player.getInternalPlayer(); - if (videoElement && videoElement.audioTracks && videoElement.audioTracks.length > 0) { - if (onAudioStatusChange) { - onAudioStatusChange(true); - } + const hasAudio = Boolean(videoElement && videoElement.audioTracks && videoElement.audioTracks.length > 0); + if (onAudioStatusChange) { + onAudioStatusChange(hasAudio); } } else { // on other platforms, inspect audio tracks before hls.js changes things const hlsPlayer = player.getInternalPlayer('hls'); From 162e81afb456e25500602e78a831726bf2ef535d Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:30:06 -0700 Subject: [PATCH 4/5] Reapply "fix: video drive the timeline (#640)" (#665) This reverts commit 81ad983f44cbeff9d899a49293dcba38f16fff8e. --- src/actions/index.js | 3 +- src/actions/types.js | 4 +- src/components/DriveMap/index.jsx | 10 +- src/components/DriveVideo/index.jsx | 239 ++++++++++++--------------- src/components/DriveView/Media.jsx | 46 +++--- src/components/TimeDisplay/index.jsx | 64 +++---- src/components/Timeline/index.jsx | 42 +++-- src/components/explorer.jsx | 10 +- src/initialState.js | 2 +- src/timeline/index.js | 33 ---- src/timeline/playback.js | 106 ++++-------- src/timeline/playback.test.js | 107 +++++------- src/timeline/segments.test.js | 4 +- src/timeline/videoPlayer.js | 75 +++++++++ src/utils/browser.js | 4 - src/utils/index.js | 6 +- 16 files changed, 340 insertions(+), 415 deletions(-) delete mode 100644 src/timeline/index.js create mode 100644 src/timeline/videoPlayer.js diff --git a/src/actions/index.js b/src/actions/index.js index d8f4b62c..d9e52096 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -5,7 +5,7 @@ import { athena as Athena, billing as Billing, devices as Devices, drives as Dri import MyCommaAuth from '@commaai/my-comma-auth'; import * as Types from './types'; -import { resetPlayback, selectLoop } from '../timeline/playback'; +import { selectLoop } from '../timeline/playback'; import {hasRoutesData } from '../timeline/segments'; import { getDeviceFromState, deviceVersionAtLeast, deviceIsOnline } from '../utils'; import { webrtcConnectionManager } from '../utils/webrtc'; @@ -163,7 +163,6 @@ export function urlForState(dongleId, log_id, start, end, prime) { function updateTimeline(state, dispatch, log_id, start, end, allowPathChange) { if (!state.loop || !state.loop.startTime || !state.loop.duration || state.loop.startTime < start || state.loop.startTime + state.loop.duration > end || state.loop.duration < end - start) { - dispatch(resetPlayback()); dispatch(selectLoop(start, end)); } diff --git a/src/actions/types.js b/src/actions/types.js index 0bbe668f..2637c526 100644 --- a/src/actions/types.js +++ b/src/actions/types.js @@ -7,7 +7,7 @@ export const ACTION_STARTUP_DATA = 'ACTION_STARTUP_DATA'; // global state management export const ACTION_SELECT_DEVICE = 'ACTION_SELECT_DEVICE'; export const ACTION_SELECT_TIME_FILTER = 'ACTION_SELECT_TIME_FILTER'; -export const ACTION_UPDATE_ROUTE_LIMIT = 'ACTION_UPDATE_ROUTE_LIMIT' +export const ACTION_UPDATE_ROUTE_LIMIT = 'ACTION_UPDATE_ROUTE_LIMIT'; export const ACTION_UPDATE_DEVICES = 'ACTION_UPDATE_DEVICES'; export const ACTION_UPDATE_DEVICE = 'ACTION_UPDATE_DEVICE'; export const ACTION_UPDATE_ROUTE = 'ACTION_UPDATE_ROUTE'; @@ -26,9 +26,11 @@ export const ACTION_PRIME_SUBSCRIBE_INFO = 'ACTION_PRIME_SUBSCRIBE_INFO'; export const ACTION_SEEK = 'action_seek'; export const ACTION_PAUSE = 'action_pause'; export const ACTION_PLAY = 'action_play'; +export const ACTION_PLAYBACK_SPEED = 'action_playback_speed'; export const ACTION_LOOP = 'action_loop'; export const ACTION_BUFFER_VIDEO = 'action_buffer_video'; export const ACTION_RESET = 'action_reset'; +export const ACTION_HAS_AUDIO = 'action_has_audio'; // segments export const ACTION_UPDATE_SEGMENT_RANGE = 'update_segment_range'; diff --git a/src/components/DriveMap/index.jsx b/src/components/DriveMap/index.jsx index 7abcc0ac..3507ef66 100644 --- a/src/components/DriveMap/index.jsx +++ b/src/components/DriveMap/index.jsx @@ -6,7 +6,6 @@ import raf from 'raf'; import ReactMapGL, { LinearInterpolator } from 'react-map-gl'; import { fetchDriveCoords } from '../../actions/cached'; -import { currentOffset } from '../../timeline'; import { DEFAULT_LOCATION, MAPBOX_STYLE, MAPBOX_TOKEN } from '../../utils/geocode'; const INTERACTION_TIMEOUT = 5000; @@ -46,7 +45,7 @@ class DriveMap extends Component { } componentDidUpdate(prevProps) { - const { dispatch, currentRoute, startTime } = this.props; + const { dispatch, currentRoute } = this.props; const prevRoute = prevProps.currentRoute?.fullname || null; const route = currentRoute?.fullname || null; @@ -57,10 +56,6 @@ class DriveMap extends Component { } } - if (prevProps.startTime && prevProps.startTime !== startTime) { - this.shouldFlyTo = true; - } - if (currentRoute && prevProps.currentRoute && currentRoute.driveCoords && prevProps.currentRoute.driveCoords !== currentRoute.driveCoords) { this.shouldFlyTo = false; @@ -99,7 +94,7 @@ class DriveMap extends Component { const markerSource = this.map && this.map.getMap().getSource('seekPoint'); if (markerSource) { if (this.props.currentRoute && this.props.currentRoute.driveCoords) { - const pos = this.posAtOffset(currentOffset()); + const pos = this.posAtOffset(this.props.offset); if (pos && pos.some((coordinate, index) => coordinate != this.lastMapPos[index])) { this.lastMapPos = pos; markerSource.setData({ @@ -310,7 +305,6 @@ class DriveMap extends Component { const stateToProps = Obstruction({ offset: 'offset', currentRoute: 'currentRoute', - startTime: 'startTime', }); export default connect(stateToProps)(DriveMap); diff --git a/src/components/DriveVideo/index.jsx b/src/components/DriveVideo/index.jsx index b35f95b7..3b0ea79b 100644 --- a/src/components/DriveVideo/index.jsx +++ b/src/components/DriveVideo/index.jsx @@ -2,7 +2,6 @@ import React, { Component } from 'react'; import { connect } from 'react-redux'; import { CircularProgress, Typography } from '@material-ui/core'; -import debounce from 'debounce'; import Obstruction from 'obstruction'; import ReactPlayer from 'react-player/file'; @@ -10,9 +9,9 @@ import { video as Video } from '@commaai/api'; import Colors from '../../colors'; import { ErrorOutline } from '../../icons'; -import { currentOffset } from '../../timeline'; -import { seek, bufferVideo } from '../../timeline/playback'; -import { isIos, isFirefox } from '../../utils/browser.js'; +import { bufferVideo, setPlaybackSpeed, resetPlayback, play, pause, seek } from '../../timeline/playback'; +import { setVideoPlayer, seekVideoPlayer, getVideoPlayerCurrentTime } from '../../timeline/videoPlayer'; +import { isIos } from '../../utils/browser.js'; const VideoOverlay = ({ loading, error }) => { let content; @@ -37,34 +36,18 @@ const VideoOverlay = ({ loading, error }) => { ); }; -const getVideoState = (videoPlayer) => { - const currentTime = videoPlayer.getCurrentTime(); - const { buffered } = videoPlayer.getInternalPlayer(); - - let bufferRemaining = -1; - for (let i = 0; i < buffered.length; i++) { - const end = buffered.end(i); - if (currentTime >= buffered.start(i) && currentTime <= end) { - bufferRemaining = end - currentTime; - break; - } - } - - return { - bufferRemaining, - hasLoaded: bufferRemaining > 0, - }; -}; - class DriveVideo extends Component { constructor(props) { super(props); this.onVideoBuffering = this.onVideoBuffering.bind(this); + this.onVideoBufferEnd = this.onVideoBufferEnd.bind(this); + this.onVideoPlay = this.onVideoPlay.bind(this); + this.onVideoPause = this.onVideoPause.bind(this); this.onHlsError = this.onHlsError.bind(this); this.onVideoError = this.onVideoError.bind(this); - this.onVideoResume = this.onVideoResume.bind(this); - this.syncVideo = debounce(this.syncVideo.bind(this), 200, true); + this.onVideoPlaybackRateChange = this.onVideoPlaybackRateChange.bind(this); + this.onTimeUpdate = this.onTimeUpdate.bind(this); this.firstSeek = true; this.videoPlayer = React.createRef(); @@ -76,44 +59,46 @@ class DriveVideo extends Component { } componentDidMount() { - const { playSpeed } = this.props; + const { dispatch } = this.props; + dispatch(resetPlayback()); if (this.videoPlayer.current) { - this.videoPlayer.current.playbackRate = playSpeed || 1; + this.videoPlayer.current.playbackRate = 1; } + setVideoPlayer(this.videoPlayer.current); this.updateVideoSource({}); - this.syncVideo(); - this.videoSyncIntv = setInterval(this.syncVideo, 500); } componentDidUpdate(prevProps) { + const videoPlayer = this.videoPlayer.current; + setVideoPlayer(videoPlayer); this.updateVideoSource(prevProps); - this.syncVideo(); } componentWillUnmount() { - if (this.videoSyncIntv) { - clearTimeout(this.videoSyncIntv); - this.videoSyncIntv = null; - } + setVideoPlayer(null); } onVideoBuffering() { - const { dispatch, currentRoute } = this.props; - const videoPlayer = this.videoPlayer.current; - if (!videoPlayer || !currentRoute || !videoPlayer.getDuration()) { - dispatch(bufferVideo(true)); - } + const { dispatch } = this.props; + dispatch(bufferVideo(true)); + } - if (this.firstSeek) { - this.firstSeek = false; - videoPlayer.seekTo(this.currentVideoTime(), 'seconds'); - } + onVideoBufferEnd() { + const { dispatch } = this.props; + const { videoError } = this.state; + if (videoError) this.setState({ videoError: null }); + dispatch(bufferVideo(false)); + } + + onVideoPlay() { + const { dispatch } = this.props; + dispatch(play()); + dispatch(bufferVideo(false)); + } - const { hasLoaded } = getVideoState(videoPlayer); - const { readyState } = videoPlayer.getInternalPlayer(); - if (!hasLoaded || readyState < 2) { - dispatch(bufferVideo(true)); - } + onVideoPause() { + const { dispatch } = this.props; + dispatch(pause()); } /** @@ -131,7 +116,7 @@ class DriveVideo extends Component { if (e.type === 'networkError' && (e.response?.code === 404)) { this.setState({ videoError: 'This video segment has not uploaded yet or has been deleted.' }); } else { - this.setState({ videoError: 'Unable to load video' }); + this.setState({ videoError: e.message }); } } @@ -155,6 +140,13 @@ class DriveVideo extends Component { return; } + if (e.name === 'NotAllowedError') { + // autoplay was blocked (e.g. iOS after backgrounding/returning to the app) + const { dispatch } = this.props; + dispatch(bufferVideo(false)); + return; + } + if (e.target?.src?.startsWith(window.location.origin) && e.target.src.endsWith('undefined')) { // TODO: figure out why the src isn't set properly // Sometimes an error will be thrown because we try to play @@ -174,13 +166,31 @@ class DriveVideo extends Component { const videoError = e.response?.code === 404 ? 'This video segment has not uploaded yet or has been deleted.' - : (e.response?.text || 'Unable to load video'); + : (e.response?.text || e.message || 'Unable to load video'); this.setState({ videoError }); } - onVideoResume() { - const { videoError } = this.state; - if (videoError) this.setState({ videoError: null }); + onVideoPlaybackRateChange(rate) { + const { dispatch } = this.props; + dispatch(setPlaybackSpeed(rate)); + } + + onTimeUpdate(event) { + const { currentRoute, loop, dispatch } = this.props; + if (!currentRoute) { + return; + } + + const videoTime = getVideoPlayerCurrentTime(currentRoute); + if (videoTime >= loop.startTime + loop.duration) { + seekVideoPlayer(loop.startTime, currentRoute); + return; + } else if (videoTime < loop.startTime) { + seekVideoPlayer(loop.startTime, currentRoute); + return; + } + + dispatch(seek(videoTime)); } updateVideoSource(prevProps) { @@ -196,66 +206,10 @@ class DriveVideo extends Component { if (src === '' || !prevProps.currentRoute || prevProps.currentRoute?.fullname !== currentRoute.fullname) { src = Video.getQcameraStreamUrl(currentRoute.fullname, currentRoute.share_exp, currentRoute.share_sig); this.setState({ src, videoError: null }); - this.syncVideo(); - } - } - - syncVideo() { - const { dispatch, isBufferingVideo, isMuted } = this.props; - const videoPlayer = this.videoPlayer.current; - if (!videoPlayer || !videoPlayer.getInternalPlayer() || !videoPlayer.getDuration()) { - return; - } - - let { desiredPlaySpeed: newPlaybackRate } = this.props; - const desiredVideoTime = this.currentVideoTime(); - const curVideoTime = videoPlayer.getCurrentTime(); - const timeDiff = desiredVideoTime - curVideoTime; - - if (Math.abs(timeDiff) <= Math.max(0.1, 0.5 * newPlaybackRate)) { // newPlaybackRate = 0 when paused, set minimum 0.1 to prevent seeking when paused - if (!isIos()) { - newPlaybackRate = Math.max(0, newPlaybackRate + Math.round(timeDiff * 10) / 10); - } - } else if (desiredVideoTime === 0 && timeDiff < 0 && curVideoTime !== videoPlayer.getDuration()) { - // logs start earlier than video, so skip to video ts 0 - dispatch(seek(currentOffset() - (timeDiff * 1000))); - } else { - videoPlayer.seekTo(desiredVideoTime, 'seconds'); - } - // most browsers don't support more than 16x playback rate, firefox mutes audio above 8x causing audio to cut in and out with timeDiff rate shifts - newPlaybackRate = Math.max(0, Math.min((isFirefox() && !isMuted) ? 8 : 16, newPlaybackRate)); - - const internalPlayer = videoPlayer.getInternalPlayer(); - - const { hasLoaded } = getVideoState(videoPlayer); - if (isBufferingVideo && internalPlayer.readyState >= 4) { - dispatch(bufferVideo(false)); - } else if (isBufferingVideo || !hasLoaded || internalPlayer.readyState < 2) { - if (!isBufferingVideo) { - dispatch(bufferVideo(true)); - } - newPlaybackRate = 0; // in some circumstances, iOS won't update readyState unless temporarily paused - } - - if (videoPlayer.getInternalPlayer('hls')) { - if (!internalPlayer.paused && newPlaybackRate === 0) { - internalPlayer.pause(); - } else if (internalPlayer.playbackRate !== newPlaybackRate && newPlaybackRate !== 0) { - internalPlayer.playbackRate = newPlaybackRate; - } - if (internalPlayer.paused && newPlaybackRate !== 0) { - const playRes = internalPlayer.play(); - if (playRes) { - playRes.catch(() => console.debug('[DriveVideo] play interrupted by pause')); - } - } - } else { - // TODO: fix iOS bug where video doesn't stop buffering while paused - internalPlayer.playbackRate = newPlaybackRate; } } - currentVideoTime(offset = currentOffset()) { + currentVideoTime(offset = this.props.offset) { const { currentRoute } = this.props; if (!currentRoute) { return 0; @@ -271,10 +225,19 @@ class DriveVideo extends Component { } render() { - const { desiredPlaySpeed, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted } = this.props; + const { isPlaying, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted } = this.props; const { src, videoError } = this.state; const onPlayerReady = (player) => { + if (this.firstSeek) { + const video = player.getInternalPlayer(); + const startSeconds = this.currentVideoTime( + this.props.loop?.startTime || 0 + ); + video.currentTime = startSeconds; + this.firstSeek = false; + } + if (isIos()) { // ios does not support hls.js and on other browsers hls.js does not directly play the m3u8 so audioTracks are not visible const videoElement = player.getInternalPlayer(); if (videoElement && videoElement.audioTracks && videoElement.audioTracks.length > 0) { @@ -295,29 +258,33 @@ class DriveVideo extends Component { }; return ( -
- - +
+
+ + +
); } @@ -325,12 +292,12 @@ class DriveVideo extends Component { const stateToProps = Obstruction({ dongleId: 'dongleId', - desiredPlaySpeed: 'desiredPlaySpeed', + isPlaying: 'isPlaying', offset: 'offset', - startTime: 'startTime', isBufferingVideo: 'isBufferingVideo', routes: 'routes', currentRoute: 'currentRoute', + loop: 'loop', }); export default connect(stateToProps)(DriveVideo); diff --git a/src/components/DriveView/Media.jsx b/src/components/DriveView/Media.jsx index ca91f726..384f5a31 100644 --- a/src/components/DriveView/Media.jsx +++ b/src/components/DriveView/Media.jsx @@ -17,7 +17,7 @@ import ResizeHandler from '../ResizeHandler'; import TimeDisplay from '../TimeDisplay'; import UploadQueue from '../Files/UploadQueue'; import SwitchLoading from '../utils/SwitchLoading'; -import { bufferVideo } from '../../timeline/playback'; +import { bufferVideo, setHasAudio } from '../../timeline/playback'; import Colors from '../../colors'; import { InfoOutline } from '../../icons'; import { deviceIsOnline, deviceOnCellular, getSegmentNumber } from '../../utils'; @@ -223,7 +223,6 @@ class Media extends Component { dcamUploadInfo: null, routePreserved: null, isMuted: true, - hasAudio: false, }; this.handleMuteToggle = this.handleMuteToggle.bind(this); @@ -251,7 +250,7 @@ class Media extends Component { } handleAudioStatusChange(hasAudio) { - this.setState({ hasAudio }); + this.props.dispatch(setHasAudio(hasAudio)); } componentDidMount() { @@ -298,12 +297,12 @@ class Media extends Component { } async copySegmentName() { - const { currentRoute } = this.props; + const { currentRoute, offset } = this.props; if (!currentRoute || !navigator.clipboard) { return; } - await navigator.clipboard.writeText(`${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute)}`); + await navigator.clipboard.writeText(`${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute, offset)}`); this.setState({ moreInfoMenu: null }); } @@ -339,7 +338,7 @@ class Media extends Component { } async uploadFile(type) { - const { dongleId, currentRoute } = this.props; + const { dongleId, currentRoute, offset } = this.props; if (!currentRoute) { return; } @@ -349,7 +348,7 @@ class Media extends Component { })); const routeNoDongleId = currentRoute.fullname.split('|')[1]; - const fileName = `${dongleId}|${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${type}`; + const fileName = `${dongleId}|${routeNoDongleId}--${getSegmentNumber(currentRoute, offset)}/${type}`; const uploading = {}; uploading[fileName] = { requested: true }; @@ -360,7 +359,7 @@ class Media extends Component { // request all possible file names for (const fn of FILE_NAMES[type]) { - const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${fn}`; + const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute, offset)}/${fn}`; paths.push(path); url_promises.push(fetchUploadUrls(dongleId, [path]).then(urls => urls[0])); } @@ -522,7 +521,8 @@ class Media extends Component { render() { const { classes } = this.props; - const { inView, windowWidth, isMuted, hasAudio } = this.state; + const { inView, windowWidth, isMuted } = this.state; + const { hasAudio } = this.props; if (this.props.menusOnly) { // for test return this.renderMenus(true); @@ -530,26 +530,24 @@ class Media extends Component { const showMapAlways = windowWidth >= 1536; const mediaContainerStyle = showMapAlways ? { width: '60%' } : { width: '100%' }; - const mapContainerStyle = showMapAlways - ? { width: '40%', marginBottom: 62, marginTop: 46, paddingLeft: 24 } - : { width: '100%' }; return (
this.setState({ windowWidth: ww })} />
{this.renderMediaOptions(showMapAlways)} - {inView === MediaType.VIDEO && ( +
+ {/* always mounted -> keeps playing + driving the clock, even under the map */} - )} - {(inView === MediaType.MAP && !showMapAlways) && ( -
- -
- )} + {!showMapAlways && ( +
+ +
+ )} +
{(inView === MediaType.VIDEO && showMapAlways) && ( -
+
)} @@ -617,7 +615,7 @@ class Media extends Component { } renderMenus(alwaysOpen = false) { - const { currentRoute, device, classes, files, profile } = this.props; + const { currentRoute, device, classes, files, offset, profile } = this.props; const { downloadMenu, moreInfoMenu, uploadModal, windowWidth, dcamUploadInfo, routePreserved } = this.state; if (!device) { @@ -627,7 +625,7 @@ class Media extends Component { let fcam = {}; let ecam = {}; let dcam = {}; let rlog = {}; if (files && currentRoute) { - const seg = `${currentRoute.fullname}--${getSegmentNumber(currentRoute)}`; + const seg = `${currentRoute.fullname}--${getSegmentNumber(currentRoute, offset)}`; fcam = files[`${seg}/cameras`] || {}; ecam = files[`${seg}/ecameras`] || {}; dcam = files[`${seg}/dcameras`] || {}; @@ -758,7 +756,7 @@ class Media extends Component { onClick={ this.copySegmentName } style={{ fontSize: windowWidth > 400 ? '0.8rem' : '0.7rem' }} > -
{ currentRoute ? `${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute)}` : '---' }
+
{ currentRoute ? `${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute, offset)}` : '---' }
{ typeof navigator.share !== 'undefined' @@ -895,10 +893,12 @@ const stateToProps = Obstruction({ device: 'device', routes: 'routes', currentRoute: 'currentRoute', + offset: 'offset', loop: 'loop', filter: 'filter', files: 'files', profile: 'profile', + hasAudio: 'hasAudio', isBufferingVideo: 'isBufferingVideo', }); diff --git a/src/components/TimeDisplay/index.jsx b/src/components/TimeDisplay/index.jsx index 1a340b1a..7dd7d4a8 100644 --- a/src/components/TimeDisplay/index.jsx +++ b/src/components/TimeDisplay/index.jsx @@ -12,8 +12,7 @@ import VolumeOff from '@material-ui/icons/VolumeOff'; import { Tooltip } from '@material-ui/core'; import { DownArrow, Forward10, Pause, PlayArrow, Replay10, UpArrow } from '../../icons'; -import { currentOffset } from '../../timeline'; -import { seek, play, pause } from '../../timeline/playback'; +import { seekVideoPlayer, playVideo, pauseVideo, setVideoPlaybackRate, isVideoPaused } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; import { isIos } from '../../utils/browser.js'; @@ -101,16 +100,6 @@ const styles = (theme) => ({ }); class TimeDisplay extends Component { - static getDerivedStateFromProps(props, state) { - if (props.desiredPlaySpeed !== 0 && props.desiredPlaySpeed !== state.desiredPlaySpeed) { - return { - ...state, - desiredPlaySpeed: props.desiredPlaySpeed, - }; - } - return state; - } - constructor(props) { super(props); @@ -124,7 +113,6 @@ class TimeDisplay extends Component { this.jumpForward = this.jumpForward.bind(this); this.state = { - desiredPlaySpeed: 1, displayTime: this.getDisplayTime(), }; } @@ -139,14 +127,13 @@ class TimeDisplay extends Component { } getDisplayTime() { - const offset = currentOffset(); - const { currentRoute } = this.props; + const { currentRoute, offset } = this.props; const now = new Date(offset + currentRoute.start_time_utc_millis); if (Number.isNaN(now.getTime())) { return '...'; } let dateString = dayjs(now).format('HH:mm:ss'); - const seg = getSegmentNumber(currentRoute); + const seg = getSegmentNumber(currentRoute, offset); if (seg !== null) { dateString = `${dateString} \u2013 ${seg}`; } @@ -155,11 +142,15 @@ class TimeDisplay extends Component { } jumpBack(amount) { - this.props.dispatch(seek(currentOffset() - amount)); + const { currentRoute } = this.props; + const offset = this.props.offset - amount; + seekVideoPlayer(offset, currentRoute); } jumpForward(amount) { - this.props.dispatch(seek(currentOffset() + amount)); + const { currentRoute } = this.props; + const offset = this.props.offset + amount; + seekVideoPlayer(offset, currentRoute); } updateTime() { @@ -176,18 +167,18 @@ class TimeDisplay extends Component { } decreaseSpeed() { - const { dispatch } = this.props; - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); } curIndex = Math.max(0, curIndex - 1); - dispatch(play(timerSteps[curIndex])); + const newSpeed = timerSteps[curIndex]; + setVideoPlaybackRate(newSpeed); } canDecreaseSpeed() { - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); @@ -196,18 +187,18 @@ class TimeDisplay extends Component { } increaseSpeed() { - const { dispatch } = this.props; - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); } curIndex = Math.min(timerSteps.length - 1, curIndex + 1); - dispatch(play(timerSteps[curIndex])); + const newSpeed = timerSteps[curIndex]; + setVideoPlaybackRate(newSpeed); } canIncreaseSpeed() { - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); @@ -216,19 +207,16 @@ class TimeDisplay extends Component { } togglePause() { - const { desiredPlaySpeed, dispatch } = this.props; - if (desiredPlaySpeed === 0) { - // eslint-disable-next-line react/destructuring-assignment - dispatch(play(this.state.desiredPlaySpeed)); + if (isVideoPaused()) { + playVideo(); } else { - dispatch(pause()); + pauseVideo(); } } render() { - const { classes, zoom, desiredPlaySpeed: videoPlaySpeed, isThin, onMuteToggle, isMuted, hasAudio } = this.props; - const { displayTime, desiredPlaySpeed } = this.state; - const isPaused = videoPlaySpeed === 0; + const { classes, zoom, isThin, onMuteToggle, isMuted, hasAudio, desiredPlaySpeed, isPlaying } = this.props; + const { displayTime } = this.state; const isExpandedCls = zoom ? 'isExpanded' : ''; const isThinCls = isThin ? 'isThin' : ''; return ( @@ -302,9 +290,9 @@ class TimeDisplay extends Component {
- {isPaused + {!isPlaying ? () : ()} @@ -316,8 +304,10 @@ class TimeDisplay extends Component { const stateToProps = Obstruction({ currentRoute: 'currentRoute', + offset: 'offset', zoom: 'zoom', - desiredPlaySpeed: 'desiredPlaySpeed' + desiredPlaySpeed: 'desiredPlaySpeed', + isPlaying: 'isPlaying', }); export default connect(stateToProps)(withStyles(styles)(TimeDisplay)); diff --git a/src/components/Timeline/index.jsx b/src/components/Timeline/index.jsx index a3b7c66d..63aeb9c0 100644 --- a/src/components/Timeline/index.jsx +++ b/src/components/Timeline/index.jsx @@ -15,9 +15,9 @@ import Thumbnails from './thumbnails'; import theme from '../../theme'; import { pushTimelineRange } from '../../actions'; import Colors from '../../colors'; -import { currentOffset } from '../../timeline'; -import { seek } from '../../timeline/playback'; +import { getVideoPlayerCurrentTime, seekVideoPlayer } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; +import { isIos } from '../../utils/browser.js'; const styles = () => ({ base: { @@ -168,6 +168,9 @@ class Timeline extends Component { this.dragBar = React.createRef(); this.hoverBead = React.createRef(); + this.currentOffset = null; + this.lastOffset = null; + const { zoomOverride, zoom } = this.props; this.state = { dragging: null, @@ -201,7 +204,8 @@ class Timeline extends Component { const { dragging } = this.state; if (!dragging || Math.abs(dragging[1] - dragging[0]) <= 3) { const percent = percentFromPointerEvent(ev); - this.props.dispatch(seek(this.percentToOffset(percent))); + const offset = this.percentToOffset(percent); + seekVideoPlayer(offset, this.props.route); } } @@ -233,7 +237,7 @@ class Timeline extends Component { } handlePointerUp(ev) { - const { route } = this.props; + const { offset, route } = this.props; // prevent preventDefault for back(3) and forward(4) mouse buttons if (ev.button !== 3 && ev.button !== 4) { @@ -255,9 +259,8 @@ class Timeline extends Component { const endOffset = Math.round(this.percentToOffset(endPercent)); if (Math.abs(dragging[1] - dragging[0]) > 3) { - const offset = currentOffset(); if (offset < startOffset || offset > endOffset) { - this.props.dispatch(seek(startOffset)); + seekVideoPlayer(startOffset, route) } const { dispatch } = this.props; const startTime = startOffset; @@ -281,20 +284,23 @@ class Timeline extends Component { } getOffset() { - if (!this.mounted) { - return; - } - raf(this.getOffset); - let offset = currentOffset(); - if (this.seekIndex) { - offset = this.seekIndex; + let offset; + if (this.props.hasAudio && isIos()) { + // video with audio doesn't report currentTime properly so we must use onTimeUpdate reported time + offset = this.props.offset; + } else { + offset = getVideoPlayerCurrentTime(this.props.route); + if (offset === null) { + offset = this.props.offset; + } } - offset = Math.floor(offset); - const percent = this.offsetToPercent(offset); + let percent = this.offsetToPercent(offset); + if (percent >= 1) percent = 1; if (this.rulerRemaining.current && this.rulerRemaining.current.parentElement) { this.rulerRemaining.current.style.left = `${Math.floor(10000 * percent) / 100}%`; this.rulerRemaining.current.style.width = `${100 - Math.floor(10000 * percent) / 100}%`; } + raf(this.getOffset); } percentToOffset(perc) { @@ -447,8 +453,14 @@ class Timeline extends Component { } const stateToProps = Obstruction({ + offset: 'offset', zoom: 'zoom', loop: 'loop', + desiredPlaySpeed: 'desiredPlaySpeed', + isBufferingVideo: 'isBufferingVideo', + currentRoute: 'currentRoute', + isPlaying: 'isPlaying', + hasAudio: 'hasAudio', }); export default connect(stateToProps)(withStyles(styles)(Timeline)); diff --git a/src/components/explorer.jsx b/src/components/explorer.jsx index f0532ae3..9d38f5b2 100644 --- a/src/components/explorer.jsx +++ b/src/components/explorer.jsx @@ -18,7 +18,6 @@ import BodyTeleop from './BodyTeleop'; import { analyticsEvent, selectDevice, updateDevice, checkLastRoutesData, streamNav } from '../actions'; import init from '../actions/startup'; import Colors from '../colors'; -import { play, pause } from '../timeline/playback'; import { verifyPairToken, pairErrorToMessage } from '../utils'; import ResizeHandler from './ResizeHandler'; @@ -141,19 +140,12 @@ class ExplorerApp extends Component { } componentDidUpdate(prevProps, prevState) { - const { pathname, zoom, dongleId, limit } = this.props; + const { pathname, dongleId, limit } = this.props; if (prevProps.pathname !== pathname) { this.setState({ drawerIsOpen: false }); } - if (!prevProps.zoom && zoom) { - this.props.dispatch(play()); - } - if (prevProps.zoom && !zoom) { - this.props.dispatch(pause()); - } - // this is necessary when user goes to explorer for the first time, dongleId is not populated in state yet // so init() will not successfully fetch routes data // when checkLastRoutesData is called within init(), it would set limit so we don't need to check again diff --git a/src/initialState.js b/src/initialState.js index 0e3a71cc..6efe9571 100644 --- a/src/initialState.js +++ b/src/initialState.js @@ -15,8 +15,8 @@ export default { desiredPlaySpeed: 1, // speed set by user isBufferingVideo: true, // if we're currently buffering for more data + isPlaying: true, // if the video is currently playing offset: null, // in miliseconds, relative to state.zoom.start - startTime: Date.now(), // millisecond timestamp in which play began routes: null, routesMeta: { diff --git a/src/timeline/index.js b/src/timeline/index.js deleted file mode 100644 index 22724754..00000000 --- a/src/timeline/index.js +++ /dev/null @@ -1,33 +0,0 @@ -import store from '../store'; - -/** - * Get current playback offset - * - * @param {object} state - * @returns {number} - */ -export function currentOffset(state = null) { - if (!state) { - state = store.getState(); - } - - /** @type {number} */ - let offset; - if (state.offset === null && state.loop?.startTime) { - offset = state.loop.startTime; - } else { - const playSpeed = state.isBufferingVideo ? 0 : state.desiredPlaySpeed; - offset = state.offset + ((Date.now() - state.startTime) * playSpeed); - } - - if (offset !== null && state.loop?.startTime) { - // respect the loop - const loopOffset = state.loop.startTime; - if (offset < loopOffset) { - offset = loopOffset; - } else if (offset > loopOffset + state.loop.duration) { - offset = ((offset - loopOffset) % state.loop.duration) + loopOffset; - } - } - return offset; -} \ No newline at end of file diff --git a/src/timeline/playback.js b/src/timeline/playback.js index acb381f5..cfe581e0 100644 --- a/src/timeline/playback.js +++ b/src/timeline/playback.js @@ -1,47 +1,31 @@ -// basic helper functions for controlling playback -// we shouldn't want to edit the raw state most of the time, helper functions are better import * as Types from '../actions/types'; -import { currentOffset } from '.'; export function reducer(_state, action) { let state = { ..._state }; - let loopOffset = null; - if (state.loop && state.loop.startTime !== null) { - loopOffset = state.loop.startTime; - } switch (action.type) { case Types.ACTION_SEEK: state = { ...state, offset: action.offset, - startTime: Date.now(), }; - - if (loopOffset !== null) { - if (state.offset < loopOffset) { - state.offset = loopOffset; - } else if (state.offset > (loopOffset + state.loop.duration)) { - state.offset = loopOffset + state.loop.duration; - } - } break; - case Types.ACTION_PAUSE: + case Types.ACTION_PLAYBACK_SPEED: state = { ...state, - offset: currentOffset(state), - startTime: Date.now(), - desiredPlaySpeed: 0, + desiredPlaySpeed: action.speed, }; break; case Types.ACTION_PLAY: - if (action.speed !== state.desiredPlaySpeed) { - state = { - ...state, - offset: currentOffset(state), - desiredPlaySpeed: action.speed, - startTime: Date.now(), - }; - } + state = { + ...state, + isPlaying: true, + }; + break; + case Types.ACTION_PAUSE: + state = { + ...state, + isPlaying: false, + }; break; case Types.ACTION_LOOP: if (action.start !== null && action.start !== undefined && action.end !== null && action.end !== undefined) { @@ -57,51 +41,25 @@ export function reducer(_state, action) { state = { ...state, isBufferingVideo: action.buffering, - offset: currentOffset(state), - startTime: Date.now(), }; break; case Types.ACTION_RESET: state = { ...state, - desiredPlaySpeed: 1, - isBufferingVideo: true, offset: 0, - startTime: Date.now(), + hasAudio: false, + }; + break; + case Types.ACTION_HAS_AUDIO: + state = { + ...state, + hasAudio: action.hasAudio, }; break; default: break; } - if (state.currentRoute && state.currentRoute.videoStartOffset && state.loop && state.zoom - && state.loop.startTime === state.zoom.start && state.zoom.start === 0) { - const loopRouteOffset = state.loop.startTime - state.zoom.start; - if (state.currentRoute.videoStartOffset > loopRouteOffset) { - state.loop = { - startTime: state.zoom.start + state.currentRoute.videoStartOffset, - duration: state.loop.duration - (state.currentRoute.videoStartOffset - loopRouteOffset), - }; - } - } - - // normalize over loop - if (state.offset !== null && state.loop?.startTime) { - const playSpeed = state.isBufferingVideo ? 0 : state.desiredPlaySpeed; - const offset = state.offset + (Date.now() - state.startTime) * playSpeed; - loopOffset = state.loop.startTime; - // has loop, trap offset within the loop - if (offset < loopOffset) { - state.startTime = Date.now(); - state.offset = loopOffset; - } else if (offset > loopOffset + state.loop.duration) { - state.offset = ((offset - loopOffset) % state.loop.duration) + loopOffset; - state.startTime = Date.now(); - } - } - - state.isBufferingVideo = Boolean(state.isBufferingVideo); - return state; } @@ -113,19 +71,20 @@ export function seek(offset) { }; } -// pause the playback -export function pause() { +// change playback speed without changing play/pause state +export function setPlaybackSpeed(speed) { return { - type: Types.ACTION_PAUSE, + type: Types.ACTION_PLAYBACK_SPEED, + speed, }; } -// resume / change play speed -export function play(speed = 1) { - return { - type: Types.ACTION_PLAY, - speed, - }; +export function play() { + return { type: Types.ACTION_PLAY }; +} + +export function pause() { + return { type: Types.ACTION_PAUSE }; } export function selectLoop(start, end) { @@ -149,3 +108,10 @@ export function resetPlayback() { type: Types.ACTION_RESET, }; } + +export function setHasAudio(hasAudio) { + return { + type: Types.ACTION_HAS_AUDIO, + hasAudio, + }; +} diff --git a/src/timeline/playback.test.js b/src/timeline/playback.test.js index f363f208..08ccedc7 100644 --- a/src/timeline/playback.test.js +++ b/src/timeline/playback.test.js @@ -1,7 +1,12 @@ /* eslint-env jest */ -import { asyncSleep } from '../utils'; -import { currentOffset } from '.'; -import { bufferVideo, pause, play, reducer, seek, selectLoop } from './playback'; +import { + bufferVideo, + reducer, + resetPlayback, + seek, + selectLoop, + setPlaybackSpeed, +} from './playback'; const makeDefaultStruct = function makeDefaultStruct() { return { @@ -13,104 +18,70 @@ const makeDefaultStruct = function makeDefaultStruct() { }; }; -// make Date.now super stable for tests -let mostRecentNow = Date.now(); -const oldNow = Date.now; -Date.now = function now() { - return mostRecentNow; -}; -function newNow() { - mostRecentNow = oldNow(); - return mostRecentNow; -} - describe('playback', () => { - it('has playback controls', async () => { - newNow(); + it('has playback controls', () => { let state = makeDefaultStruct(); - // should do nothing - state = reducer(state, pause()); + // stop playback + state = reducer(state, setPlaybackSpeed(0)); expect(state.desiredPlaySpeed).toEqual(0); - // start playing, should set start time and such - let playTime = newNow(); - state = reducer(state, play()); - // this is a (usually 1ms) race condition - expect(state.startTime).toEqual(playTime); + // start playing + state = reducer(state, setPlaybackSpeed(1)); expect(state.desiredPlaySpeed).toEqual(1); - await asyncSleep(100 + Math.random() * 200); - // should update offset - let ellapsed = newNow() - playTime; - state = reducer(state, pause()); - - expect(state.offset).toEqual(ellapsed); - - // start playing, should set start time and such - playTime = newNow(); - state = reducer(state, play(0.5)); - // this is a (usually 1ms) race condition - expect(state.startTime).toEqual(playTime); - expect(state.desiredPlaySpeed).toEqual(0.5); + // seek updates offset + state = reducer(state, seek(123)); + expect(state.offset).toEqual(123); - await asyncSleep(100 + Math.random() * 200); - // should update offset, playback speed 1/2 - ellapsed += (newNow() - playTime) / 2; - expect(currentOffset(state)).toEqual(ellapsed); - state = reducer(state, pause()); + // reset clears offset + state = reducer(state, resetPlayback()); + expect(state.offset).toEqual(0); + }); - expect(state.offset).toEqual(ellapsed); + it('should set loop start time and duration', () => { + let state = makeDefaultStruct(); - // seek! - newNow(); - state = reducer(state, seek(123)); - expect(state.offset).toEqual(123); - expect(state.startTime).toEqual(Date.now()); - expect(currentOffset(state)).toEqual(123); + state = reducer(state, selectLoop( + 1000, + 2000, + )); + expect(state.loop.startTime).toEqual(1000); + expect(state.loop.duration).toEqual(1000); }); - it('should clamp loop when seeked after loop end time', () => { - newNow(); + it('should not clamp offset when seeked after loop end time', () => { let state = makeDefaultStruct(); - // set up loop - state = reducer(state, play()); state = reducer(state, selectLoop( 1000, 2000, )); expect(state.loop.startTime).toEqual(1000); - // seek past loop end boundary a state = reducer(state, seek(3000)); expect(state.loop.startTime).toEqual(1000); - expect(state.offset).toEqual(2000); + expect(state.offset).toEqual(3000); }); - it('should clamp loop when seeked before loop start time', () => { - newNow(); + it('should not clamp offset when seeked before loop start time', () => { let state = makeDefaultStruct(); - // set up loop - state = reducer(state, play()); state = reducer(state, selectLoop( 1000, 2000, )); expect(state.loop.startTime).toEqual(1000); - // seek past loop end boundary a state = reducer(state, seek(0)); expect(state.loop.startTime).toEqual(1000); - expect(state.offset).toEqual(1000); + expect(state.offset).toEqual(0); }); - it('should buffer video and data', async () => { - newNow(); + it('should buffer video and data', () => { let state = makeDefaultStruct(); - state = reducer(state, play()); + state = reducer(state, setPlaybackSpeed(1)); expect(state.desiredPlaySpeed).toEqual(1); // claim the video is buffering @@ -118,17 +89,13 @@ describe('playback', () => { expect(state.desiredPlaySpeed).toEqual(1); expect(state.isBufferingVideo).toEqual(true); - state = reducer(state, play(0.5)); + state = reducer(state, setPlaybackSpeed(0.5)); expect(state.desiredPlaySpeed).toEqual(0.5); expect(state.isBufferingVideo).toEqual(true); - expect(state.desiredPlaySpeed).toEqual(0.5); - - state = reducer(state, play(2)); + state = reducer(state, setPlaybackSpeed(2)); state = reducer(state, bufferVideo(false)); expect(state.desiredPlaySpeed).toEqual(2); expect(state.isBufferingVideo).toEqual(false); - - expect(state.desiredPlaySpeed).toEqual(2); }); -}); \ No newline at end of file +}); diff --git a/src/timeline/segments.test.js b/src/timeline/segments.test.js index edb58671..70a824c2 100644 --- a/src/timeline/segments.test.js +++ b/src/timeline/segments.test.js @@ -29,7 +29,7 @@ const routes = [{ describe('segments', () => { it('finds current segment', async () => { const [route] = routes; - expect(getSegmentNumber(route)).toBe(0); + expect(getSegmentNumber(route, 0)).toBe(0); }); it('can check if it has segment metadata', () => { @@ -97,4 +97,4 @@ describe('segments', () => { dongleId: 'asdfasdf', })).toBe(true); }); -}); \ No newline at end of file +}); diff --git a/src/timeline/videoPlayer.js b/src/timeline/videoPlayer.js new file mode 100644 index 00000000..15da5221 --- /dev/null +++ b/src/timeline/videoPlayer.js @@ -0,0 +1,75 @@ +let videoPlayer = null; + +export function setVideoPlayer(player) { + videoPlayer = player; +} + +function getInternal() { + if (!videoPlayer || !videoPlayer.getInternalPlayer) { + return null; + } + return videoPlayer.getInternalPlayer(); +} + +export function seekVideoPlayer(offset, route) { + if (!videoPlayer || !videoPlayer.getInternalPlayer || !videoPlayer.getDuration()) { + return false; + } + const internal = getInternal(); + + let videoTime = offset; + if (route && route.videoStartOffset) { + videoTime -= route.videoStartOffset; + } + videoTime = Math.max(0, videoTime / 1000); + + internal.currentTime = videoTime; + return true; +} + +export function getVideoPlayerCurrentTime(route) { + const internal = getInternal(); + if (!internal) { + return null; + } + const videoStartOffset = (route && route.videoStartOffset) || 0; + return internal.currentTime * 1000 + videoStartOffset; +} + +export function isVideoPaused() { + const internal = getInternal(); + if (!internal) { + return true; + } + return internal.paused; +} + +export function playVideo() { + const internal = getInternal(); + if (!internal) { + return false; + } + const promise = internal.play(); + if (promise && typeof promise.catch === 'function') { + promise.catch(() => {}); + } + return true; +} + +export function pauseVideo() { + const internal = getInternal(); + if (!internal) { + return false; + } + internal.pause(); + return true; +} + +export function setVideoPlaybackRate(rate) { + const internal = getInternal(); + if (!internal) { + return false; + } + internal.playbackRate = rate; + return true; +} diff --git a/src/utils/browser.js b/src/utils/browser.js index cd4030e5..35dbbf79 100644 --- a/src/utils/browser.js +++ b/src/utils/browser.js @@ -1,7 +1,3 @@ export function isIos() { return /iphone|ipad|ipod/i.test(navigator.userAgent); } - -export function isFirefox() { - return navigator.userAgent.toLowerCase().includes('firefox'); -} diff --git a/src/utils/index.js b/src/utils/index.js index ac8b6717..f1c560fa 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -3,8 +3,6 @@ import dayjs from 'dayjs'; import relativeTime from 'dayjs/plugin/relativeTime'; import decodeJwt, { InvalidTokenError } from 'jwt-decode'; -import { currentOffset } from '../timeline'; - dayjs.extend(relativeTime); export const emptyDevice = { @@ -193,8 +191,8 @@ export function getSegmentNumber(route, offset) { if (!route) { return null; } - if (offset === undefined) { - offset = currentOffset(); + if (offset === undefined || offset === null) { + return null; } return Math.floor(offset / (60*1000)); From 61eda69c8b41ea45188d8a5d21ea48d8826e81bd Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:34:06 -0700 Subject: [PATCH 5/5] map smoothness, allow seeking when video fails --- src/App.test.jsx | 22 ++++++++++- src/actions/cached.js | 1 + src/actions/types.js | 1 + src/api/demo.js | 25 +++++++------ src/components/DriveMap/index.jsx | 24 ++++++++++-- src/components/DriveVideo/index.jsx | 56 ++++++++++------------------ src/components/TimeDisplay/index.jsx | 16 ++++++-- src/components/Timeline/index.jsx | 24 +++++++++--- src/initialState.js | 2 + src/timeline/playback.js | 20 ++++++++++ 10 files changed, 129 insertions(+), 62 deletions(-) diff --git a/src/App.test.jsx b/src/App.test.jsx index cb53f4ec..4ac637be 100644 --- a/src/App.test.jsx +++ b/src/App.test.jsx @@ -6,7 +6,7 @@ import App from './App'; import { createInitialState } from './initialState'; import { createAppStore } from './store'; -const mocks = vi.hoisted(() => ({ authenticated: true, options: {}, requests: [], hardNavigate: vi.fn() })); +const mocks = vi.hoisted(() => ({ authenticated: true, options: {}, requests: [], hardNavigate: vi.fn(), playerProps: null })); vi.mock('@commaai/my-comma-auth', () => ({ default: { @@ -41,6 +41,7 @@ vi.mock('react-map-gl', () => ({ })); vi.mock('react-player/file', () => ({ default: React.forwardRef((_props, ref) => { + mocks.playerProps = _props; React.useImperativeHandle(ref, () => ({ getCurrentTime: () => 0, getDuration: () => 60, @@ -154,6 +155,7 @@ describe('whole-app behavior', () => { localStorage.clear(); sessionStorage.clear(); mocks.hardNavigate.mockClear(); + mocks.playerProps = null; }); test('root uses a valid stored device and keeps the selection', async () => { @@ -271,4 +273,22 @@ describe('whole-app behavior', () => { fireEvent.click(within(document.body).getByRole('button', { name: 'Close' })); await waitFor(() => expect(history.location.pathname).toBe(`/${FIRST}`)); }); + + test('a failed video leaves timeline navigation owned by Redux', async () => { + const { store } = await renderApp(`/${FIRST}/${LOG}`); + const timeline = await screen.findByRole('slider', { name: 'Drive timeline' }); + + act(() => { + mocks.playerProps.onError('hlsError', { + fatal: true, + type: 'networkError', + response: { code: 404 }, + }); + }); + expect(store.getState().videoStatus).toBe('failed'); + + fireEvent.pointerDown(timeline, { button: 0, clientX: 500, pageX: 500 }); + fireEvent.pointerUp(timeline, { button: 0, clientX: 500, pageX: 500 }); + expect(store.getState().offset).toBe(30000); + }); }); diff --git a/src/actions/cached.js b/src/actions/cached.js index aec49074..6d060ef5 100644 --- a/src/actions/cached.js +++ b/src/actions/cached.js @@ -1,6 +1,7 @@ import * as Sentry from '@sentry/react'; import * as Types from './types'; +import { api } from '../api/backend'; import { reverseLookup } from '../utils/geocode'; const USE_LOCAL_COORDS_DATA = import.meta.env.VITE_APP_LOCAL_COORDS_DATA === 'true'; diff --git a/src/actions/types.js b/src/actions/types.js index ede7d6c2..f816cbf2 100644 --- a/src/actions/types.js +++ b/src/actions/types.js @@ -31,6 +31,7 @@ export const ACTION_LOOP = 'ACTION_LOOP'; export const ACTION_BUFFER_VIDEO = 'ACTION_BUFFER_VIDEO'; export const ACTION_RESET = 'ACTION_RESET'; export const ACTION_HAS_AUDIO = 'ACTION_HAS_AUDIO'; +export const ACTION_VIDEO_STATUS = 'ACTION_VIDEO_STATUS'; // segments export const ACTION_UPDATE_SEGMENT_RANGE = 'ACTION_UPDATE_SEGMENT_RANGE'; diff --git a/src/api/demo.js b/src/api/demo.js index 5de89a49..ea11efae 100644 --- a/src/api/demo.js +++ b/src/api/demo.js @@ -32,8 +32,6 @@ const DEMO_PROFILE = { const AFFECTED_SEGMENT = 1; -// One clone per test case. Each case mutates a fresh clone of the real public -// data on its way into the frontend. const MISSING_DATA_CASES = [ { title: 'Epoch date/time (no clock)', @@ -102,16 +100,21 @@ const MISSING_DATA_CASES = [ }, ]; -// Keep two full-length routes for every case: one where the whole route is -// affected and one where only a single segment is affected. -const TEST_CASES = MISSING_DATA_CASES.flatMap((testCase) => [ - testCase, + +const TEST_CASES = [ { - ...testCase, - title: `${testCase.title} (1 segment)`, - affectedSegment: AFFECTED_SEGMENT, + title: 'Public route (no issues)', + route() {}, }, -]); + ...MISSING_DATA_CASES.flatMap((testCase) => [ + testCase, + { + ...testCase, + title: `${testCase.title} (1 segment)`, + affectedSegment: AFFECTED_SEGMENT, + }, + ]), +]; function fileSegmentNumber(file) { const pathParts = new URL(file).pathname.split('/'); @@ -168,7 +171,7 @@ export function createDemoBackend(realBackend) { } // Clone the cached public route into fresh demo routes on every call, each - // with a unique demo route ID and one mutation per test case. + // with a unique demo route ID and its test case's mutation, if any. async function listDemoRoutes(routeStr) { const publicRoute = await fetchPublicRoute(); const routes = TEST_CASES.map((testCase, index) => { diff --git a/src/components/DriveMap/index.jsx b/src/components/DriveMap/index.jsx index 5b460c24..11b84bca 100644 --- a/src/components/DriveMap/index.jsx +++ b/src/components/DriveMap/index.jsx @@ -4,6 +4,9 @@ import { connect } from 'react-redux'; import ReactMapGL, { LinearInterpolator } from 'react-map-gl'; import { fetchDriveCoords } from '../../actions/cached'; +import { VideoStatus } from '../../timeline/playback'; +import { getVideoPlayerCurrentTime } from '../../timeline/videoPlayer'; +import { isIos } from '../../utils/browser'; import { DEFAULT_LOCATION, MAPBOX_STYLE, MAPBOX_TOKEN } from '../../utils/geocode'; const INTERACTION_TIMEOUT = 5000; @@ -68,6 +71,10 @@ class DriveMap extends Component { componentWillUnmount() { this.mounted = false; + if (this.rafId) { + cancelAnimationFrame(this.rafId); + this.rafId = null; + } } onInteraction(ev) { @@ -92,7 +99,16 @@ class DriveMap extends Component { const markerSource = this.map && this.map.getMap().getSource('seekPoint'); if (markerSource) { if (this.props.currentRoute && this.props.currentRoute.driveCoords) { - const pos = this.posAtOffset(this.props.offset); + let offset; + if (this.props.videoStatus === VideoStatus.FAILED || (this.props.hasAudio && isIos())) { + offset = this.props.offset; + } else { + offset = getVideoPlayerCurrentTime(this.props.currentRoute); + if (offset === null) { + offset = this.props.offset; + } + } + const pos = this.posAtOffset(offset); if (pos && pos.some((coordinate, index) => coordinate != this.lastMapPos[index])) { this.lastMapPos = pos; markerSource.setData({ @@ -111,7 +127,7 @@ class DriveMap extends Component { } } - requestAnimationFrame(this.updateMarkerPos); + this.rafId = requestAnimationFrame(this.updateMarkerPos); } moveViewportTo(pos) { @@ -202,7 +218,7 @@ class DriveMap extends Component { } initMap(mapComponent) { - if (!mapComponent) { + if (!mapComponent || typeof mapComponent.getMap !== 'function') { this.map = null; return; } @@ -303,6 +319,8 @@ class DriveMap extends Component { const stateToProps = (state) => ({ offset: state.offset, currentRoute: state.currentRoute, + hasAudio: state.hasAudio, + videoStatus: state.videoStatus, }); export default connect(stateToProps)(DriveMap); diff --git a/src/components/DriveVideo/index.jsx b/src/components/DriveVideo/index.jsx index e06ac0cd..1a18dc9f 100644 --- a/src/components/DriveVideo/index.jsx +++ b/src/components/DriveVideo/index.jsx @@ -8,41 +8,12 @@ import { api } from '../../api/backend'; import Colors from '../../colors'; import { ErrorOutline } from '../../icons'; -import { bufferVideo, setPlaybackSpeed, resetPlayback, play, pause, seek } from '../../timeline/playback'; +import { + bufferVideo, setPlaybackSpeed, resetPlayback, play, pause, seek, setVideoStatus, VideoStatus, +} from '../../timeline/playback'; import { setVideoPlayer, seekVideoPlayer, getVideoPlayerCurrentTime } from '../../timeline/videoPlayer'; import { isIos } from '../../utils/browser.js'; -// Leading-edge debounce: run immediately, then ignore calls until `wait` ms after the last one. -function debounceLeading(func, wait) { - let timeout = null; - let args; - let context; - let timestamp; - - function later() { - const last = Date.now() - timestamp; - if (last < wait && last >= 0) { - timeout = setTimeout(later, wait - last); - } else { - timeout = null; - } - } - - return function debounced(...nextArgs) { - context = this; - args = nextArgs; - timestamp = Date.now(); - const callNow = !timeout; - if (!timeout) { - timeout = setTimeout(later, wait); - } - if (callNow) { - return func.apply(context, args); - } - return undefined; - }; -} - const VideoOverlay = ({ loading, error }) => { let content; if (error) { @@ -138,11 +109,15 @@ class DriveVideo extends Component { const { dispatch } = this.props; dispatch(bufferVideo(true)); - if (!e.fatal) return; - else if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { + if (!e.fatal) { + return; + } + if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { // buffer but no error return; - } else if (e.type === 'networkError' && (e.response?.code === 404)) { + } + dispatch(setVideoStatus(VideoStatus.FAILED)); + if (e.type === 'networkError' && (e.response?.code === 404)) { this.setState({ videoError: 'This video segment has not uploaded yet or has been deleted.' }); } else { const message = @@ -189,6 +164,7 @@ class DriveVideo extends Component { const { dispatch } = this.props; dispatch(bufferVideo(true)); + dispatch(setVideoStatus(VideoStatus.FAILED)); if (e.type === 'networkError') { console.error('Network error', { e, data }); @@ -214,6 +190,9 @@ class DriveVideo extends Component { } const videoTime = getVideoPlayerCurrentTime(currentRoute); + if (videoTime === null) { + return; + } if (videoTime >= loop.startTime + loop.duration) { seekVideoPlayer(loop.startTime, currentRoute); return; @@ -227,15 +206,17 @@ class DriveVideo extends Component { updateVideoSource(prevProps) { let { src } = this.state; - const { currentRoute } = this.props; + const { currentRoute, dispatch } = this.props; if (!currentRoute) { if (src !== '') { + dispatch(setVideoStatus(VideoStatus.LOADING)); this.setState({ src: '', videoError: null }); } return; } if (src === '' || !prevProps.currentRoute || prevProps.currentRoute?.fullname !== currentRoute.fullname) { + dispatch(setVideoStatus(VideoStatus.LOADING)); src = api.video.getQcameraStreamUrl(currentRoute.fullname, currentRoute.share_exp, currentRoute.share_sig); this.setState({ src, videoError: null }); this.firstSeek = true; @@ -258,7 +239,7 @@ class DriveVideo extends Component { } render() { - const { isPlaying, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted } = this.props; + const { isPlaying, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted, dispatch } = this.props; const { src, videoError } = this.state; const onPlayerReady = (player) => { @@ -270,6 +251,7 @@ class DriveVideo extends Component { video.currentTime = startSeconds; this.firstSeek = false; } + dispatch(setVideoStatus(VideoStatus.READY)); if (isIos()) { // ios does not support hls.js and on other browsers hls.js does not directly play the m3u8 so audioTracks are not visible const videoElement = player.getInternalPlayer(); diff --git a/src/components/TimeDisplay/index.jsx b/src/components/TimeDisplay/index.jsx index 660dd5d6..842fd645 100644 --- a/src/components/TimeDisplay/index.jsx +++ b/src/components/TimeDisplay/index.jsx @@ -10,6 +10,7 @@ import VolumeOff from '@material-ui/icons/VolumeOff'; import { Tooltip } from '@material-ui/core'; import { DownArrow, Forward10, Pause, PlayArrow, Replay10, UpArrow } from '../../icons'; +import { VideoStatus } from '../../timeline/playback'; import { seekVideoPlayer, playVideo, pauseVideo, setVideoPlaybackRate, isVideoPaused } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; import { isIos } from '../../utils/browser.js'; @@ -213,16 +214,20 @@ class TimeDisplay extends Component { } render() { - const { classes, zoom, isThin, onMuteToggle, isMuted, hasAudio, desiredPlaySpeed, isPlaying } = this.props; + const { + classes, zoom, isThin, onMuteToggle, isMuted, hasAudio, desiredPlaySpeed, isPlaying, videoStatus, + } = this.props; const { displayTime } = this.state; const isExpandedCls = zoom ? 'isExpanded' : ''; const isThinCls = isThin ? 'isThin' : ''; + const controlsDisabled = videoStatus === VideoStatus.FAILED; return (
this.jumpBack(10000) } + disabled={controlsDisabled} aria-label="Jump back 10 seconds" > @@ -232,6 +237,7 @@ class TimeDisplay extends Component { this.jumpForward(10000) } + disabled={controlsDisabled} aria-label="Jump forward 10 seconds" > @@ -250,7 +256,7 @@ class TimeDisplay extends Component { @@ -262,7 +268,7 @@ class TimeDisplay extends Component { @@ -275,7 +281,7 @@ class TimeDisplay extends Component { {isMuted @@ -288,6 +294,7 @@ class TimeDisplay extends Component {
{!isPlaying @@ -306,6 +313,7 @@ const stateToProps = (state) => ({ desiredPlaySpeed: state.desiredPlaySpeed, isPlaying: state.isPlaying, offset: state.offset, + videoStatus: state.videoStatus, }); export default connect(stateToProps)(withStyles(styles)(TimeDisplay)); diff --git a/src/components/Timeline/index.jsx b/src/components/Timeline/index.jsx index b9229f1d..d1b698e6 100644 --- a/src/components/Timeline/index.jsx +++ b/src/components/Timeline/index.jsx @@ -10,6 +10,7 @@ import Thumbnails from './thumbnails'; import theme from '../../theme'; import { pushTimelineRange } from '../../actions'; import Colors from '../../colors'; +import { seek, VideoStatus } from '../../timeline/playback'; import { getVideoPlayerCurrentTime, seekVideoPlayer } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; import { isIos } from '../../utils/browser.js'; @@ -153,6 +154,7 @@ class Timeline extends Component { this.handlePointerDown = this.handlePointerDown.bind(this); this.handlePointerUp = this.handlePointerUp.bind(this); this.handlePointerLeave = this.handlePointerLeave.bind(this); + this.seekToOffset = this.seekToOffset.bind(this); this.percentToOffset = this.percentToOffset.bind(this); this.segmentNum = this.segmentNum.bind(this); this.onRulerRef = this.onRulerRef.bind(this); @@ -181,7 +183,7 @@ class Timeline extends Component { componentDidMount() { this.mounted = true; - this.rafId = raf(this.getOffset); + this.rafId = requestAnimationFrame(this.getOffset); this.componentDidUpdate({}); if (typeof ResizeObserver !== 'undefined' && this.thumbnailsRef.current) { @@ -207,7 +209,7 @@ class Timeline extends Component { componentWillUnmount() { this.mounted = false; if (this.rafId) { - raf.cancel(this.rafId); + cancelAnimationFrame(this.rafId); this.rafId = null; } if (this.resizeObserver) { @@ -216,12 +218,21 @@ class Timeline extends Component { } } + seekToOffset(offset) { + const { dispatch, route, videoStatus } = this.props; + if (videoStatus === VideoStatus.FAILED) { + dispatch(seek(offset)); + return; + } + seekVideoPlayer(offset, route); + } + handleClick(ev) { const { dragging } = this.state; if (!dragging || Math.abs(dragging[1] - dragging[0]) <= 3) { const percent = percentFromPointerEvent(ev); const offset = this.percentToOffset(percent); - seekVideoPlayer(offset, this.props.route); + this.seekToOffset(offset); } } @@ -276,7 +287,7 @@ class Timeline extends Component { if (Math.abs(dragging[1] - dragging[0]) > 3) { if (offset < startOffset || offset > endOffset) { - seekVideoPlayer(startOffset, route) + this.seekToOffset(startOffset); } const { dispatch } = this.props; const startTime = startOffset; @@ -304,7 +315,7 @@ class Timeline extends Component { return; } let offset; - if (this.props.hasAudio && isIos()) { + if (this.props.videoStatus === VideoStatus.FAILED || (this.props.hasAudio && isIos())) { // video with audio doesn't report currentTime properly so we must use onTimeUpdate reported time offset = this.props.offset; } else { @@ -319,7 +330,7 @@ class Timeline extends Component { this.rulerRemaining.current.style.left = `${Math.floor(10000 * percent) / 100}%`; this.rulerRemaining.current.style.width = `${100 - Math.floor(10000 * percent) / 100}%`; } - this.rafId = raf(this.getOffset); + this.rafId = requestAnimationFrame(this.getOffset); } percentToOffset(perc) { @@ -479,6 +490,7 @@ const stateToProps = (state) => ({ currentRoute: state.currentRoute, isPlaying: state.isPlaying, hasAudio: state.hasAudio, + videoStatus: state.videoStatus, }); export default connect(stateToProps)(withStyles(styles)(Timeline)); diff --git a/src/initialState.js b/src/initialState.js index b631a6e5..864c99e4 100644 --- a/src/initialState.js +++ b/src/initialState.js @@ -1,4 +1,5 @@ import { getDongleID, getSegmentRange, getPrimeNav, getStreamNav } from './url'; +import { VideoStatus } from './timeline/playback'; export function getDefaultFilter() { const d = new Date(); @@ -17,6 +18,7 @@ export function createInitialState(pathname = window.location.pathname) { desiredPlaySpeed: 1, // speed set by user isBufferingVideo: true, // if we're currently buffering for more data isPlaying: true, // if the video is currently playing + videoStatus: VideoStatus.LOADING, offset: null, // in miliseconds, relative to state.zoom.start routes: null, diff --git a/src/timeline/playback.js b/src/timeline/playback.js index 7d0aea0f..f97a2168 100644 --- a/src/timeline/playback.js +++ b/src/timeline/playback.js @@ -1,5 +1,11 @@ import * as Types from '../actions/types'; +export const VideoStatus = { + LOADING: 'loading', + READY: 'ready', + FAILED: 'failed', +}; + export function reducer(_state, action) { let state = { ..._state }; switch (action.type) { @@ -49,6 +55,7 @@ export function reducer(_state, action) { offset: 0, desiredPlaySpeed: 1, hasAudio: false, + videoStatus: VideoStatus.LOADING, }; break; case Types.ACTION_HAS_AUDIO: @@ -57,6 +64,12 @@ export function reducer(_state, action) { hasAudio: action.hasAudio, }; break; + case Types.ACTION_VIDEO_STATUS: + state = { + ...state, + videoStatus: action.status, + }; + break; default: break; } @@ -116,3 +129,10 @@ export function setHasAudio(hasAudio) { hasAudio, }; } + +export function setVideoStatus(status) { + return { + type: Types.ACTION_VIDEO_STATUS, + status, + }; +}