From f04e711ea34ad3ad5ee642047be5d56a18fa0b53 Mon Sep 17 00:00:00 2001 From: Michael Weghorn Date: Thu, 30 May 2024 13:24:02 +0200 Subject: [PATCH] tdf#145735 qt avmedia: Implement frame grabber Add a new `QtFrameGrabber` class that implements the `css::media::XFrameGrabber` interface and return an instance of it in `QtPlayer::createFrameGrabber`. As there seems to be no direct way to retrieve the image/frame of a video at a certain point in time, create a separate `QMediaPlayer` instance for the `QtFrameGrabber`, set a video sink, connect to its `&QVideoSink::videoFrameChanged` signal and start playing the video (and stop it again once the first frame has been received) in order to retrieve a corresponding frame. From that `QVideoFrame`, a `QImage` can be retrieved that can then be converted to an `XGraphic`. With this in place, a frame from the actual video is now displaced in Impress in non-presentation mode instead of just a generic "video icon" placeholder when opening a presentation containing a video, (e.g. attachment 145517 from tdf#120452) when the qt6 VCL plugin is in use. Change-Id: I3bba3c0fb62a219ac632ceed03ec17f9078f18d0 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/168255 Tested-by: Jenkins Reviewed-by: Michael Weghorn --- avmedia/CustomTarget_avmediaqt6_moc.mk | 1 + avmedia/Library_avmediaqt6.mk | 1 + avmedia/source/qt6/QtFrameGrabber.cxx | 106 +++++++++++++++++++++++++ avmedia/source/qt6/QtFrameGrabber.hxx | 50 ++++++++++++ avmedia/source/qt6/QtPlayer.cxx | 9 ++- 5 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 avmedia/source/qt6/QtFrameGrabber.cxx create mode 100644 avmedia/source/qt6/QtFrameGrabber.hxx diff --git a/avmedia/CustomTarget_avmediaqt6_moc.mk b/avmedia/CustomTarget_avmediaqt6_moc.mk index 0f9ca1af23b6..6cce5d258323 100644 --- a/avmedia/CustomTarget_avmediaqt6_moc.mk +++ b/avmedia/CustomTarget_avmediaqt6_moc.mk @@ -10,6 +10,7 @@ $(eval $(call gb_CustomTarget_CustomTarget,avmedia/source/qt6)) $(call gb_CustomTarget_get_target,avmedia/source/qt6) : \ + $(gb_CustomTarget_workdir)/avmedia/source/qt6/QtFrameGrabber.moc \ $(gb_CustomTarget_workdir)/avmedia/source/qt6/QtPlayer.moc diff --git a/avmedia/Library_avmediaqt6.mk b/avmedia/Library_avmediaqt6.mk index a1acb7568d02..86b4553e1add 100644 --- a/avmedia/Library_avmediaqt6.mk +++ b/avmedia/Library_avmediaqt6.mk @@ -37,6 +37,7 @@ $(eval $(call gb_Library_use_libraries,avmediaqt6,\ $(eval $(call gb_Library_add_exception_objects,avmediaqt6,\ avmedia/source/qt6/gstwindow \ + avmedia/source/qt6/QtFrameGrabber \ avmedia/source/qt6/QtManager \ avmedia/source/qt6/QtPlayer \ )) diff --git a/avmedia/source/qt6/QtFrameGrabber.cxx b/avmedia/source/qt6/QtFrameGrabber.cxx new file mode 100644 index 000000000000..58726d649af0 --- /dev/null +++ b/avmedia/source/qt6/QtFrameGrabber.cxx @@ -0,0 +1,106 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include "QtFrameGrabber.hxx" +#include + +using namespace ::com::sun::star; + +namespace +{ +inline OUString toOUString(const QString& s) +{ + return OUString(reinterpret_cast(s.data()), s.length()); +} + +uno::Reference toXGraphic(const QImage& rImage) +{ + QByteArray aData; + QBuffer aBuffer(&aData); + rImage.save(&aBuffer, "PNG"); + + SvMemoryStream aStream(aData.data(), aData.size(), StreamMode::READ); + vcl::PngImageReader aReader(aStream); + Graphic aGraphic; + aReader.read(aGraphic); + + return aGraphic.GetXGraphic(); +} +} + +namespace avmedia::qt +{ +QtFrameGrabber::QtFrameGrabber(const QUrl& rSourceUrl) + : m_bWaitingForFrame(false) +{ + m_xMediaPlayer = std::make_unique(); + m_xMediaPlayer->setSource(rSourceUrl); + + m_xVideoSink = std::make_unique(); + m_xMediaPlayer->setVideoSink(m_xVideoSink.get()); + + connect(m_xMediaPlayer.get(), &QMediaPlayer::errorOccurred, this, + &QtFrameGrabber::onErrorOccured, Qt::BlockingQueuedConnection); +} + +void QtFrameGrabber::onErrorOccured(QMediaPlayer::Error eError, const QString& rErrorString) +{ + std::lock_guard aLock(m_aMutex); + + SAL_WARN("avmedia", "Media playback error occured when trying to grab frame: " + << toOUString(rErrorString) << ", code: " << eError); + + m_bWaitingForFrame = false; +} + +void QtFrameGrabber::onVideoFrameChanged(const QVideoFrame& rFrame) +{ + std::lock_guard aLock(m_aMutex); + + disconnect(m_xVideoSink.get(), &QVideoSink::videoFrameChanged, this, + &QtFrameGrabber::onVideoFrameChanged); + + const QImage aImage = rFrame.toImage(); + m_xGraphic = toXGraphic(aImage); + m_bWaitingForFrame = false; +} + +css::uno::Reference SAL_CALL QtFrameGrabber::grabFrame(double fMediaTime) +{ + std::lock_guard aLock(m_aMutex); + + m_xMediaPlayer->setPosition(fMediaTime * 1000); + + // in order to get a video frame, connect to videoFrameChanged signal and start playing + // until the first frame has been received + m_bWaitingForFrame = true; + connect(m_xVideoSink.get(), &QVideoSink::videoFrameChanged, this, + &QtFrameGrabber::onVideoFrameChanged, Qt::BlockingQueuedConnection); + m_xMediaPlayer->play(); + while (m_bWaitingForFrame) + Scheduler::ProcessEventsToIdle(); + m_xMediaPlayer->stop(); + + uno::Reference xGraphic = m_xGraphic; + m_xGraphic.clear(); + return xGraphic; +} + +}; // namespace + +/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */ diff --git a/avmedia/source/qt6/QtFrameGrabber.hxx b/avmedia/source/qt6/QtFrameGrabber.hxx new file mode 100644 index 000000000000..125ec74fc381 --- /dev/null +++ b/avmedia/source/qt6/QtFrameGrabber.hxx @@ -0,0 +1,50 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace avmedia::qt +{ +class QtFrameGrabber : public QObject, public ::cppu::WeakImplHelper +{ + Q_OBJECT + +private: + std::unique_ptr m_xVideoSink; + std::unique_ptr m_xMediaPlayer; + + std::recursive_mutex m_aMutex; + bool m_bWaitingForFrame; + css::uno::Reference m_xGraphic; + +public: + QtFrameGrabber(const QUrl& rSourceUrl); + + virtual css::uno::Reference + SAL_CALL grabFrame(double fMediaTime) override; + +private slots: + void onErrorOccured(QMediaPlayer::Error eError, const QString& rErrorString); + void onVideoFrameChanged(const QVideoFrame& rFrame); +}; + +} // namespace avmedia::qt + +/* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */ diff --git a/avmedia/source/qt6/QtPlayer.cxx b/avmedia/source/qt6/QtPlayer.cxx index 5f75394ff1a3..d5291d5b0d5a 100644 --- a/avmedia/source/qt6/QtPlayer.cxx +++ b/avmedia/source/qt6/QtPlayer.cxx @@ -27,6 +27,7 @@ #include #include +#include "QtFrameGrabber.hxx" #include "QtPlayer.hxx" #include @@ -229,7 +230,13 @@ uno::Reference<::media::XPlayerWindow> return xRet; } -uno::Reference SAL_CALL QtPlayer::createFrameGrabber() { return nullptr; } +uno::Reference SAL_CALL QtPlayer::createFrameGrabber() +{ + osl::MutexGuard aGuard(m_aMutex); + + rtl::Reference xFrameGrabber = new QtFrameGrabber(m_xMediaPlayer->source()); + return xFrameGrabber; +} void SAL_CALL QtPlayer::addPlayerListener(const css::uno::Reference& rListener)