/* This file is part of Telegram Desktop, the official desktop application for the Telegram messaging service. For license and copyright information please follow this link: https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL */ #include "media/streaming/media_streaming_player.h" #include "media/streaming/media_streaming_file.h" #include "media/streaming/media_streaming_loader.h" #include "media/streaming/media_streaming_audio_track.h" #include "media/streaming/media_streaming_video_track.h" #include "media/audio/media_audio.h" // for SupportsSpeedControl() #include "data/data_document.h" // for DocumentData::duration() namespace Media { namespace Streaming { namespace { constexpr auto kReceivedTillEnd = std::numeric_limits::max(); constexpr auto kBufferFor = 3 * crl::time(1000); constexpr auto kLoadInAdvanceFor = 64 * crl::time(1000); constexpr auto kMsFrequency = 1000; // 1000 ms per second. // If we played for 3 seconds and got stuck it looks like we're loading // slower than we're playing, so load full file in that case. constexpr auto kLoadFullIfStuckAfterPlayback = 3 * crl::time(1000); [[nodiscard]] crl::time TrackClampReceivedTill( crl::time position, const TrackState &state) { return (state.duration == kTimeUnknown || position == kTimeUnknown) ? position : (position == kReceivedTillEnd) ? state.duration : std::clamp(position, crl::time(0), state.duration - 1); } [[nodiscard]] bool FullTrackReceived(const TrackState &state) { return (state.duration != kTimeUnknown) && (state.receivedTill == state.duration); } void SaveValidStateInformation(TrackState &to, TrackState &&from) { Expects(from.position != kTimeUnknown); Expects(from.receivedTill != kTimeUnknown); Expects(from.duration != kTimeUnknown); to.duration = from.duration; to.position = from.position; to.receivedTill = (to.receivedTill == kTimeUnknown) ? from.receivedTill : std::clamp( std::max(from.receivedTill, to.receivedTill), to.position, to.duration); } void SaveValidAudioInformation( AudioInformation &to, AudioInformation &&from) { SaveValidStateInformation(to.state, std::move(from.state)); } void SaveValidVideoInformation( VideoInformation &to, VideoInformation &&from) { Expects(!from.size.isEmpty()); Expects(!from.cover.isNull()); SaveValidStateInformation(to.state, std::move(from.state)); to.size = from.size; to.cover = std::move(from.cover); to.rotation = from.rotation; } void SaveValidStartInformation(Information &to, Information &&from) { if (from.audio.state.duration != kTimeUnknown) { SaveValidAudioInformation(to.audio, std::move(from.audio)); } if (from.video.state.duration != kTimeUnknown) { SaveValidVideoInformation(to.video, std::move(from.video)); } } } // namespace Player::Player( not_null owner, std::unique_ptr loader) : _file(std::make_unique(owner, std::move(loader))) , _renderFrameTimer([=] { checkNextFrame(); }) { } not_null Player::delegate() { return static_cast(this); } void Player::checkNextFrame() { Expects(_nextFrameTime != kTimeUnknown); Expects(!_renderFrameTimer.isActive()); const auto now = crl::now(); if (now < _nextFrameTime) { _renderFrameTimer.callOnce(_nextFrameTime - now); } else { renderFrame(now); } } void Player::renderFrame(crl::time now) { Expects(_video != nullptr); const auto position = _video->markFrameDisplayed(now); Assert(position != kTimeUnknown); videoPlayedTill(position); } template void Player::trackReceivedTill( const Track &track, TrackState &state, crl::time position) { if (position == kTimeUnknown) { return; } else if (state.duration != kTimeUnknown) { position = std::clamp(position, crl::time(0), state.duration); if (state.receivedTill < position) { state.receivedTill = position; trackSendReceivedTill(track, state); } } else { state.receivedTill = position; } if (!_pauseReading && bothReceivedEnough(kLoadInAdvanceFor) && !receivedTillEnd()) { _pauseReading = true; } } static auto wakes = 0; template void Player::trackPlayedTill( const Track &track, TrackState &state, crl::time position) { const auto guard = base::make_weak(&_sessionGuard); trackReceivedTill(track, state, position); if (guard && position != kTimeUnknown) { position = std::clamp(position, crl::time(0), state.duration); state.position = position; _updates.fire({ PlaybackUpdate{ position } }); } if (_pauseReading && (!bothReceivedEnough(kLoadInAdvanceFor) || receivedTillEnd())) { _pauseReading = false; _file->wake(); ++wakes; } } template void Player::trackSendReceivedTill( const Track &track, TrackState &state) { Expects(state.duration != kTimeUnknown); Expects(state.receivedTill != kTimeUnknown); _updates.fire({ PreloadedUpdate{ state.receivedTill } }); } void Player::audioReceivedTill(crl::time position) { Expects(_audio != nullptr); position = TrackClampReceivedTill(position, _information.audio.state); trackReceivedTill(*_audio, _information.audio.state, position); checkResumeFromWaitingForData(); } void Player::audioPlayedTill(crl::time position) { Expects(_audio != nullptr); trackPlayedTill(*_audio, _information.audio.state, position); } void Player::videoReceivedTill(crl::time position) { Expects(_video != nullptr); position = TrackClampReceivedTill(position, _information.video.state); trackReceivedTill(*_video, _information.video.state, position); } void Player::videoPlayedTill(crl::time position) { Expects(_video != nullptr); trackPlayedTill(*_video, _information.video.state, position); } void Player::fileReady(Stream &&video, Stream &&audio) { _waitingForData = false; const auto weak = base::make_weak(&_sessionGuard); const auto ready = [=](const Information &data) { crl::on_main(weak, [=, data = data]() mutable { streamReady(std::move(data)); }); }; const auto error = [&](auto &stream) { return [=, &stream] { crl::on_main(weak, [=, &stream] { stream = nullptr; streamFailed(); }); }; }; const auto mode = _options.mode; if (audio.codec && (mode == Mode::Audio || mode == Mode::Both)) { if (_options.audioId.audio() != nullptr) { _audioId = AudioMsgId( _options.audioId.audio(), _options.audioId.contextId(), AudioMsgId::CreateExternalPlayId()); } else { _audioId = AudioMsgId::ForVideo(); } _audio = std::make_unique( _options, std::move(audio), _audioId, ready, error(_audio)); } else { _audioId = AudioMsgId(); } if (video.codec && (mode == Mode::Video || mode == Mode::Both)) { _video = std::make_unique( _options, std::move(video), _audioId, ready, error(_video)); } if ((mode == Mode::Audio && !_audio) || (mode == Mode::Video && !_video) || (!_audio && !_video)) { LOG(("Streaming Error: Required stream not found for mode %1." ).arg(int(mode))); fileError(); } } void Player::fileError() { _waitingForData = false; crl::on_main(&_sessionGuard, [=] { fail(); }); } void Player::fileWaitingForData() { if (_waitingForData) { return; } _waitingForData = true; if (_audio) { _audio->waitForData(); } if (_video) { _video->waitForData(); } } bool Player::fileProcessPacket(Packet &&packet) { _waitingForData = false; const auto &native = packet.fields(); const auto index = native.stream_index; if (packet.empty()) { _readTillEnd = true; if (_audio) { crl::on_main(&_sessionGuard, [=] { audioReceivedTill(kReceivedTillEnd); }); _audio->process(Packet()); } if (_video) { crl::on_main(&_sessionGuard, [=] { videoReceivedTill(kReceivedTillEnd); }); _video->process(Packet()); } } else if (_audio && _audio->streamIndex() == native.stream_index) { const auto time = PacketPosition(packet, _audio->streamTimeBase()); crl::on_main(&_sessionGuard, [=] { audioReceivedTill(time); }); _audio->process(std::move(packet)); } else if (_video && _video->streamIndex() == native.stream_index) { const auto time = PacketPosition(packet, _video->streamTimeBase()); crl::on_main(&_sessionGuard, [=] { videoReceivedTill(time); }); _video->process(std::move(packet)); } return fileReadMore(); } bool Player::fileReadMore() { // return true if looping. return !_readTillEnd && !_pauseReading; } void Player::streamReady(Information &&information) { SaveValidStartInformation(_information, std::move(information)); provideStartInformation(); } void Player::streamFailed() { if (_stage == Stage::Initializing) { provideStartInformation(); } else { fail(); } } void Player::provideStartInformation() { Expects(_stage == Stage::Initializing); if ((_audio && _information.audio.state.duration == kTimeUnknown) || (_video && _information.video.state.duration == kTimeUnknown)) { return; // Not ready yet. } else if ((!_audio && !_video) || (!_audio && _options.mode == Mode::Audio) || (!_video && _options.mode == Mode::Video)) { fail(); } else { _stage = Stage::Ready; // Don't keep the reference to the video cover. auto copy = _information; _information.video.cover = QImage(); _updates.fire(Update{ std::move(copy) }); if (_stage == Stage::Ready && !_paused) { _paused = true; updatePausedState(); } } } void Player::fail() { _sessionLifetime = rpl::lifetime(); const auto stopGuarded = crl::guard(&_sessionGuard, [=] { stop(); }); _lastFailureStage = _stage; _updates.fire_error({}); stopGuarded(); } void Player::play(const PlaybackOptions &options) { Expects(options.speed >= 0.5 && options.speed <= 2.); stop(); _lastFailureStage = Stage::Uninitialized; _options = options; if (!Media::Audio::SupportsSpeedControl()) { _options.speed = 1.; } _stage = Stage::Initializing; _file->start(delegate(), _options.position); } void Player::pause() { Expects(active()); _pausedByUser = true; updatePausedState(); } void Player::resume() { Expects(active()); _pausedByUser = false; updatePausedState(); } void Player::updatePausedState() { const auto paused = _pausedByUser || _pausedByWaitingForData; if (_paused == paused) { return; } _paused = paused; if (!_paused && _stage == Stage::Ready) { const auto guard = base::make_weak(&_sessionGuard); start(); if (!guard) { return; } } if (_stage != Stage::Started) { return; } if (_paused) { _pausedTime = crl::now(); //if (_pausedByWaitingForData // && _pausedTime - _startedTime > kLoadFullIfStuckAfterPlayback) { // _loadFull = true; //} if (_audio) { _audio->pause(_pausedTime); } if (_video) { _video->pause(_pausedTime); } } else { _startedTime = crl::now(); if (_audio) { _audio->resume(_startedTime); } if (_video) { _video->resume(_startedTime); } } } bool Player::trackReceivedEnough( const TrackState &state, crl::time amount) const { return FullTrackReceived(state) || (state.position != kTimeUnknown && state.position + amount <= state.receivedTill); } bool Player::bothReceivedEnough(crl::time amount) const { auto &info = _information; return (!_audio || trackReceivedEnough(info.audio.state, amount)) && (!_video || trackReceivedEnough(info.video.state, amount)); } bool Player::receivedTillEnd() const { return (!_video || FullTrackReceived(_information.video.state)) && (!_audio || FullTrackReceived(_information.audio.state)); } void Player::checkResumeFromWaitingForData() { if (_pausedByWaitingForData && bothReceivedEnough(kBufferFor)) { _pausedByWaitingForData = false; updatePausedState(); _updates.fire({ WaitingForData{ false } }); } } void Player::start() { Expects(_stage == Stage::Ready); _stage = Stage::Started; const auto guard = base::make_weak(&_sessionGuard); rpl::merge( _audio ? _audio->waitingForData() : rpl::never(), _video ? _video->waitingForData() : rpl::never() ) | rpl::filter([=] { return !receivedTillEnd(); }) | rpl::start_with_next([=] { _pausedByWaitingForData = true; updatePausedState(); _updates.fire({ WaitingForData{ true } }); }, _sessionLifetime); if (guard && _audio) { _audio->playPosition( ) | rpl::start_with_next_done([=](crl::time position) { audioPlayedTill(position); }, [=] { Expects(_stage == Stage::Started); _audioFinished = true; if (!_video || _videoFinished) { _updates.fire({ Finished() }); } }, _sessionLifetime); } if (guard && _video) { _video->renderNextFrame( ) | rpl::start_with_next_done([=](crl::time when) { _nextFrameTime = when; checkNextFrame(); }, [=] { Expects(_stage == Stage::Started); _videoFinished = true; if (!_audio || _audioFinished) { _updates.fire({ Finished() }); } }, _sessionLifetime); } if (guard && _audio) { trackSendReceivedTill(*_audio, _information.audio.state); } if (guard && _video) { trackSendReceivedTill(*_video, _information.video.state); } } void Player::stop() { _file->stop(); _sessionLifetime = rpl::lifetime(); _stage = Stage::Uninitialized; _audio = nullptr; _video = nullptr; invalidate_weak_ptrs(&_sessionGuard); _pausedByUser = _pausedByWaitingForData = _paused = false; _renderFrameTimer.cancel(); _audioFinished = false; _videoFinished = false; _pauseReading = false; _readTillEnd = false; _information = Information(); } bool Player::failed() const { return (_lastFailureStage != Stage::Uninitialized); } bool Player::playing() const { return (_stage == Stage::Started) && !paused() && !finished() && !failed(); } bool Player::buffering() const { return _pausedByWaitingForData; } bool Player::paused() const { return _pausedByUser; } bool Player::finished() const { return (_stage == Stage::Started) && (!_audio || _audioFinished) && (!_video || _videoFinished); } void Player::setSpeed(float64 speed) { Expects(active()); Expects(speed >= 0.5 && speed <= 2.); if (!Media::Audio::SupportsSpeedControl()) { speed = 1.; } if (_options.speed != speed) { _options.speed = speed; if (_audio) { _audio->setSpeed(speed); } if (_video) { _video->setSpeed(speed); } } } bool Player::active() const { return (_stage != Stage::Uninitialized) && !finished() && !failed(); } bool Player::ready() const { return (_stage != Stage::Uninitialized) && (_stage != Stage::Initializing); } rpl::producer Player::updates() const { return _updates.events(); } QImage Player::frame(const FrameRequest &request) const { Expects(_video != nullptr); return _video->frame(request); } Media::Player::TrackState Player::prepareLegacyState() const { using namespace Media::Player; auto result = Media::Player::TrackState(); result.id = _audioId.externalPlayId() ? _audioId : _options.audioId; result.state = (_lastFailureStage == Stage::Started) ? State::StoppedAtError : failed() ? State::StoppedAtStart : finished() ? State::StoppedAtEnd : paused() ? State::Paused : State::Playing; result.position = std::max( _information.audio.state.position, _information.video.state.position); if (result.position == kTimeUnknown) { result.position = _options.position; } result.length = std::max( _information.audio.state.duration, _information.video.state.duration); if (result.length == kTimeUnknown && _options.audioId.audio()) { const auto document = _options.audioId.audio(); const auto duration = document->song() ? document->song()->duration : document->duration(); if (duration > 0) { result.length = duration * crl::time(1000); } else { result.length = std::max(result.position, crl::time(0)); } } result.frequency = kMsFrequency; return result; } rpl::lifetime &Player::lifetime() { return _lifetime; } Player::~Player() { // The order of field destruction is important. // // We are forced to maintain the correct order in the stop() method, // because it can be called even before the player destruction. // // So instead of maintaining it in the class definition as well we // simply call stop() here, after that the destruction is trivial. LOG(("WAKES: %1").arg(wakes)); stop(); } } // namespace Streaming } // namespace Media