0

remoting: Support callback based frame capturing

Rather than doing a timer based frame capture, this change
modifies the implementation to rely on callbacks from webrtc
capturer whenever a new frame is received via pipewire.

Corresponding WebRTC change is here:
https://webrtc-review.googlesource.com/c/src/+/291080

Bug: chromium:1291247
Change-Id: Ia00edc110dfe40918e1ad0862472f70d7c849c9f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4188923
Reviewed-by: Lambros Lambrou <lambroslambrou@chromium.org>
Commit-Queue: Salman Malik <salmanmalik@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1099800}
This commit is contained in:
Salman
2023-02-01 13:41:22 +00:00
committed by Chromium LUCI CQ
parent 77aea12fe9
commit 2df935ea25
17 changed files with 263 additions and 14 deletions

@ -5,6 +5,11 @@
#include "remoting/host/desktop_and_cursor_conditional_composer.h"
#include "base/functional/bind.h"
#include "build/build_config.h"
#if BUILDFLAG(IS_LINUX)
#include "remoting/host/linux/wayland_utils.h"
#endif
namespace remoting {
@ -90,6 +95,19 @@ bool DesktopAndCursorConditionalComposer::IsOccluded(
return capturer_->IsOccluded(pos);
}
bool DesktopAndCursorConditionalComposer::SupportsFrameCallbacks() {
#if BUILDFLAG(IS_LINUX)
return IsRunningWayland();
#else
return false;
#endif
}
void DesktopAndCursorConditionalComposer::SetMaxFrameRate(
uint32_t max_frame_rate) {
capturer_->SetMaxFrameRate(max_frame_rate);
}
#if defined(WEBRTC_USE_GIO)
void DesktopAndCursorConditionalComposer::GetMetadataAsync(
base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {

@ -39,6 +39,8 @@ class DesktopAndCursorConditionalComposer : public DesktopCapturer {
void SetMouseCursor(
std::unique_ptr<webrtc::MouseCursor> mouse_cursor) override;
void SetMouseCursorPosition(const webrtc::DesktopVector& position) override;
bool SupportsFrameCallbacks() override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
#if defined(WEBRTC_USE_GIO)
void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
callback) override;

@ -34,6 +34,7 @@
#include "base/environment.h"
#include "base/nix/xdg_util.h"
#include "remoting/host/linux/wayland_desktop_capturer.h"
#include "remoting/host/linux/wayland_utils.h"
#endif
namespace remoting {
@ -58,12 +59,14 @@ class DesktopCapturerProxy::Core : public webrtc::DesktopCapturer::Callback {
std::unique_ptr<webrtc::SharedMemoryFactory> shared_memory_factory);
void SelectSource(SourceId id);
void CaptureFrame();
void SetMaxFrameRate(uint32_t max_frame_rate);
#if defined(WEBRTC_USE_GIO)
void GetAndSetMetadata();
#endif
private:
// webrtc::DesktopCapturer::Callback implementation.
void OnFrameCaptureStart() override;
void OnCaptureResult(webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) override;
@ -155,6 +158,14 @@ void DesktopCapturerProxy::Core::CaptureFrame() {
}
}
void DesktopCapturerProxy::Core::SetMaxFrameRate(uint32_t max_frame_rate) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (capturer_) {
capturer_->SetMaxFrameRate(max_frame_rate);
}
}
#if defined(WEBRTC_USE_GIO)
void DesktopCapturerProxy::Core::GetAndSetMetadata() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
@ -168,11 +179,15 @@ void DesktopCapturerProxy::Core::GetAndSetMetadata() {
}
#endif
void DesktopCapturerProxy::Core::OnFrameCaptureStart() {
caller_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&DesktopCapturerProxy::OnFrameCaptureStarting, proxy_));
}
void DesktopCapturerProxy::Core::OnCaptureResult(
webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
caller_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&DesktopCapturerProxy::OnFrameCaptured, proxy_,
result, std::move(frame)));
@ -253,6 +268,12 @@ bool DesktopCapturerProxy::SelectSource(SourceId id) {
return false;
}
void DesktopCapturerProxy::OnFrameCaptureStarting() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
callback_->OnFrameCaptureStart();
}
void DesktopCapturerProxy::OnFrameCaptured(
webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) {
@ -277,6 +298,24 @@ void DesktopCapturerProxy::OnMetadata(webrtc::DesktopCaptureMetadata metadata) {
std::move(metadata_callback_).Run(std::move(metadata));
}
#endif
bool DesktopCapturerProxy::SupportsFrameCallbacks() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
#if BUILDFLAG(IS_LINUX)
return IsRunningWayland();
#else
return false;
#endif
}
void DesktopCapturerProxy::SetMaxFrameRate(uint32_t max_frame_rate) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
capture_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&Core::SetMaxFrameRate,
base::Unretained(core_.get()), max_frame_rate));
}
} // namespace remoting

@ -52,6 +52,8 @@ class DesktopCapturerProxy : public DesktopCapturer {
void CaptureFrame() override;
bool GetSourceList(SourceList* sources) override;
bool SelectSource(SourceId id) override;
bool SupportsFrameCallbacks() override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
#if defined(WEBRTC_USE_GIO)
void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
callback) override;
@ -60,6 +62,7 @@ class DesktopCapturerProxy : public DesktopCapturer {
private:
class Core;
void OnFrameCaptureStarting();
void OnFrameCaptured(webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame);

@ -8,10 +8,15 @@
#include <utility>
#include "base/logging.h"
#include "build/build_config.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
#if BUILDFLAG(IS_LINUX)
#include "remoting/host/linux/wayland_utils.h"
#endif
namespace remoting {
DesktopCapturerWrapper::DesktopCapturerWrapper() {
@ -78,6 +83,12 @@ bool DesktopCapturerWrapper::SelectSource(SourceId id) {
return false;
}
void DesktopCapturerWrapper::OnFrameCaptureStart() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
callback_->OnFrameCaptureStart();
}
void DesktopCapturerWrapper::OnCaptureResult(
webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) {
@ -86,6 +97,24 @@ void DesktopCapturerWrapper::OnCaptureResult(
callback_->OnCaptureResult(result, std::move(frame));
}
bool DesktopCapturerWrapper::SupportsFrameCallbacks() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
#if BUILDFLAG(IS_LINUX)
return capturer_ && IsRunningWayland();
#else
return false;
#endif
}
void DesktopCapturerWrapper::SetMaxFrameRate(uint32_t max_frame_rate) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (capturer_) {
capturer_->SetMaxFrameRate(max_frame_rate);
}
}
#if defined(WEBRTC_USE_GIO)
void DesktopCapturerWrapper::GetMetadataAsync(
base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {

@ -37,6 +37,8 @@ class DesktopCapturerWrapper : public DesktopCapturer,
void CaptureFrame() override;
bool GetSourceList(SourceList* sources) override;
bool SelectSource(SourceId id) override;
bool SupportsFrameCallbacks() override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
#if defined(WEBRTC_USE_GIO)
void GetMetadataAsync(base::OnceCallback<void(webrtc::DesktopCaptureMetadata)>
callback) override;
@ -44,6 +46,7 @@ class DesktopCapturerWrapper : public DesktopCapturer,
private:
// webrtc::DesktopCapturer::Callback implementation.
void OnFrameCaptureStart() override;
void OnCaptureResult(webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) override;

@ -79,7 +79,7 @@ if (enable_me2me_host) {
}
}
if (remoting_use_x11) {
if (is_linux) {
source_set("wayland") {
sources = [
"wayland_utils.cc",
@ -87,6 +87,9 @@ if (remoting_use_x11) {
]
deps = [ "//base" ]
}
}
if (remoting_use_x11) {
source_set("x11") {
sources = [
"unicode_to_keysym.cc",

@ -28,7 +28,9 @@ WaylandDesktopCapturer::WaylandDesktopCapturer(
// Note: RemoteDesktopPortal doesn't own `this`
std::make_unique<xdg_portal::RemoteDesktopPortal>(
this,
options.prefer_cursor_embedded())) {}
options.prefer_cursor_embedded())) {
base_capturer_pipewire_.SendFramesImmediately(true);
}
WaylandDesktopCapturer::~WaylandDesktopCapturer() {
WaylandManager::Get()->OnDesktopCapturerDestroyed();
@ -58,6 +60,10 @@ void WaylandDesktopCapturer::SetScreenResolution(ScreenResolution resolution,
resolution.dimensions().height());
}
void WaylandDesktopCapturer::SetMaxFrameRate(uint32_t max_frame_rate) {
base_capturer_pipewire_.SetMaxFrameRate(max_frame_rate);
}
#if defined(WEBRTC_USE_GIO)
webrtc::DesktopCaptureMetadata WaylandDesktopCapturer::GetMetadata() {
SessionDetails session_details = base_capturer_pipewire_.GetSessionDetails();
@ -88,4 +94,8 @@ void WaylandDesktopCapturer::OnScreenCastSessionClosed() {
base_capturer_pipewire_.OnScreenCastSessionClosed();
}
bool WaylandDesktopCapturer::SupportsFrameCallbacks() {
return true;
}
} // namespace remoting

@ -7,6 +7,7 @@
#include "base/memory/weak_ptr.h"
#include "remoting/host/base/screen_resolution.h"
#include "remoting/protocol/desktop_capturer.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_metadata.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_capture_types.h"
@ -14,7 +15,7 @@
namespace remoting {
class WaylandDesktopCapturer : public webrtc::DesktopCapturer,
class WaylandDesktopCapturer : public DesktopCapturer,
public webrtc::ScreenCastPortal::PortalNotifier {
public:
explicit WaylandDesktopCapturer(const webrtc::DesktopCaptureOptions& options);
@ -27,6 +28,8 @@ class WaylandDesktopCapturer : public webrtc::DesktopCapturer,
void CaptureFrame() override;
bool GetSourceList(SourceList* sources) override;
bool SelectSource(SourceId id) override;
bool SupportsFrameCallbacks() override;
void SetMaxFrameRate(uint32_t max_frame_rate) override;
// PortalNotifier interface.
void OnScreenCastRequestResult(webrtc::xdg_portal::RequestResponse result,

@ -76,6 +76,7 @@ static_library("protocol") {
"data_channel_manager.cc",
"data_channel_manager.h",
"datagram_channel_factory.h",
"desktop_capturer.cc",
"desktop_capturer.h",
"display_size.cc",
"display_size.h",
@ -278,6 +279,8 @@ static_library("protocol") {
"capture_scheduler.h",
"ice_connection_to_client.cc",
"ice_connection_to_client.h",
"no_op_webrtc_frame_scheduler.cc",
"no_op_webrtc_frame_scheduler.h",
"video_frame_pump.cc",
"video_frame_pump.h",
"video_stream_event_router.cc",

@ -0,0 +1,13 @@
// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "remoting/protocol/desktop_capturer.h"
namespace remoting {
bool DesktopCapturer::SupportsFrameCallbacks() {
return false;
}
} // namespace remoting

@ -26,6 +26,10 @@ class DesktopCapturer : public webrtc::DesktopCapturer {
// Change the position of the composed mouse cursor.
virtual void SetMouseCursorPosition(const webrtc::DesktopVector& position) {}
// Whether capturer can notify the callback interface of the available frames
// immediately.
virtual bool SupportsFrameCallbacks();
#if defined(WEBRTC_USE_GIO)
virtual void GetMetadataAsync(
base::OnceCallback<void(webrtc::DesktopCaptureMetadata)> callback) {}

@ -0,0 +1,53 @@
// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "remoting/protocol/no_op_webrtc_frame_scheduler.h"
#include "base/notreached.h"
namespace remoting::protocol {
NoOpWebrtcFrameScheduler::NoOpWebrtcFrameScheduler(DesktopCapturer* capturer)
: capturer_(capturer) {
DETACH_FROM_SEQUENCE(sequence_checker_);
}
NoOpWebrtcFrameScheduler::~NoOpWebrtcFrameScheduler() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}
void NoOpWebrtcFrameScheduler::Start(
const base::RepeatingClosure& capture_callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}
void NoOpWebrtcFrameScheduler::Pause(bool pause) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (pause) {
capturer_->SetMaxFrameRate(0u);
} else {
capturer_->SetMaxFrameRate(last_frame_rate_);
}
}
void NoOpWebrtcFrameScheduler::OnFrameCaptured(
const webrtc::DesktopFrame* frame) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}
void NoOpWebrtcFrameScheduler::SetMaxFramerateFps(int max_framerate_fps) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
last_frame_rate_ = max_framerate_fps;
capturer_->SetMaxFrameRate(last_frame_rate_);
}
void NoOpWebrtcFrameScheduler::BoostCaptureRate(
base::TimeDelta capture_interval,
base::TimeDelta duration) {
NOTIMPLEMENTED() << "Boosting frame rate is not supported for wayland";
}
} // namespace remoting::protocol

@ -0,0 +1,42 @@
// Copyright 2023 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef REMOTING_PROTOCOL_NO_OP_WEBRTC_FRAME_SCHEDULER_H_
#define REMOTING_PROTOCOL_NO_OP_WEBRTC_FRAME_SCHEDULER_H_
#include "remoting/protocol/webrtc_frame_scheduler.h"
#include "base/functional/callback.h"
#include "base/sequence_checker.h"
#include "remoting/protocol/desktop_capturer.h"
namespace remoting::protocol {
// This no-op scheduler is used in place of constant frame rate scheduler
// when we have a capturer that supports sending us events whenever a frame is
// captured.
class NoOpWebrtcFrameScheduler : public WebrtcFrameScheduler {
public:
explicit NoOpWebrtcFrameScheduler(DesktopCapturer* capturer);
~NoOpWebrtcFrameScheduler() override;
// WebrtcFrameScheduler Interface.
void Start(const base::RepeatingClosure& capture_callback) override;
void Pause(bool pause) override;
void OnFrameCaptured(const webrtc::DesktopFrame* frame) override;
void SetMaxFramerateFps(int max_framerate_fps) override;
void BoostCaptureRate(base::TimeDelta capture_interval,
base::TimeDelta duration) override;
private:
base::raw_ptr<DesktopCapturer> capturer_
GUARDED_BY_CONTEXT(sequence_checker_);
uint32_t last_frame_rate_ GUARDED_BY_CONTEXT(sequence_checker_) = 60;
SEQUENCE_CHECKER(sequence_checker_);
};
} // namespace remoting::protocol
#endif // REMOTING_PROTOCOL_NO_OP_WEBRTC_FRAME_SCHEDULER_H_

@ -7,6 +7,8 @@
#include "base/functional/callback_forward.h"
#include "base/time/time.h"
namespace webrtc {
class DesktopFrame;
} // namespace webrtc
@ -35,6 +37,11 @@ class WebrtcFrameScheduler {
// Called when WebRTC requests the VideoTrackSource to provide frames at a
// maximum framerate.
virtual void SetMaxFramerateFps(int max_framerate_fps) = 0;
// Temporarily adjusts the capture rate to |capture_interval| for the next
// |duration|.
virtual void BoostCaptureRate(base::TimeDelta capture_interval,
base::TimeDelta duration) = 0;
};
} // namespace remoting::protocol

@ -36,7 +36,7 @@ class WebrtcFrameSchedulerConstantRate : public WebrtcFrameScheduler {
// Temporarily adjusts the capture rate to |capture_interval| for the next
// |duration|.
void BoostCaptureRate(base::TimeDelta capture_interval,
base::TimeDelta duration);
base::TimeDelta duration) override;
void SetPostTaskAdjustmentForTest(base::TimeDelta post_task_adjustment);

@ -18,6 +18,7 @@
#include "remoting/protocol/desktop_capturer.h"
#include "remoting/protocol/frame_stats.h"
#include "remoting/protocol/host_video_stats_dispatcher.h"
#include "remoting/protocol/no_op_webrtc_frame_scheduler.h"
#include "remoting/protocol/webrtc_frame_scheduler_constant_rate.h"
#include "remoting/protocol/webrtc_transport.h"
#include "remoting/protocol/webrtc_video_encoder_factory.h"
@ -73,6 +74,7 @@ class WebrtcVideoStream::Core : public webrtc::DesktopCapturer::Callback {
void Start();
// webrtc::DesktopCapturer::Callback interface.
void OnFrameCaptureStart() override;
void OnCaptureResult(webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) override;
@ -111,7 +113,7 @@ class WebrtcVideoStream::Core : public webrtc::DesktopCapturer::Callback {
std::unique_ptr<DesktopCapturer> capturer_;
// Schedules the next video frame.
WebrtcFrameSchedulerConstantRate scheduler_;
std::unique_ptr<WebrtcFrameScheduler> scheduler_;
// Provides event timestamps which are used for |current_frame_stats|.
scoped_refptr<InputEventTimestampsSource> event_timestamps_source_;
@ -131,6 +133,11 @@ WebrtcVideoStream::Core::Core(std::unique_ptr<DesktopCapturer> capturer,
video_stream_(std::move(video_stream)),
video_stream_task_runner_(
base::SingleThreadTaskRunner::GetCurrentDefault()) {
if (capturer_->SupportsFrameCallbacks()) {
scheduler_ = std::make_unique<NoOpWebrtcFrameScheduler>(capturer_.get());
} else {
scheduler_ = std::make_unique<WebrtcFrameSchedulerConstantRate>();
}
DETACH_FROM_THREAD(thread_checker_);
}
@ -140,10 +147,20 @@ void WebrtcVideoStream::Core::Start() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
capturer_->Start(this);
scheduler_.Start(base::BindRepeating(
scheduler_->Start(base::BindRepeating(
&WebrtcVideoStream::Core::CaptureNextFrame, base::Unretained(this)));
}
void WebrtcVideoStream::Core::OnFrameCaptureStart() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
current_frame_stats_ = std::make_unique<FrameStats>();
current_frame_stats_->capture_started_time = base::TimeTicks::Now();
current_frame_stats_->input_event_timestamps =
event_timestamps_source_->TakeLastEventTimestamps();
current_frame_stats_->screen_id = screen_id_;
}
void WebrtcVideoStream::Core::OnCaptureResult(
webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) {
@ -154,7 +171,7 @@ void WebrtcVideoStream::Core::OnCaptureResult(
base::Milliseconds(frame ? frame->capture_time_ms() : 0);
if (!frame || frame->size().is_empty()) {
scheduler_.OnFrameCaptured(nullptr);
scheduler_->OnFrameCaptured(nullptr);
return;
}
@ -173,7 +190,7 @@ void WebrtcVideoStream::Core::OnCaptureResult(
current_frame_stats_->capturer_id = frame->capturer_id();
scheduler_.OnFrameCaptured(frame.get());
scheduler_->OnFrameCaptured(frame.get());
video_stream_task_runner_->PostTask(
FROM_HERE,
@ -189,7 +206,7 @@ void WebrtcVideoStream::Core::SetEventTimestampsSource(
void WebrtcVideoStream::Core::Pause(bool pause) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
scheduler_.Pause(pause);
scheduler_->Pause(pause);
}
void WebrtcVideoStream::Core::SelectSource(webrtc::ScreenId id) {
@ -217,12 +234,12 @@ void WebrtcVideoStream::Core::SetMouseCursorPosition(
void WebrtcVideoStream::Core::BoostFramerate(base::TimeDelta capture_interval,
base::TimeDelta boost_duration) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
scheduler_.BoostCaptureRate(capture_interval, boost_duration);
scheduler_->BoostCaptureRate(capture_interval, boost_duration);
}
void WebrtcVideoStream::Core::SetMaxFramerateFps(int max_framerate_fps) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
scheduler_.SetMaxFramerateFps(max_framerate_fps);
scheduler_->SetMaxFramerateFps(max_framerate_fps);
}
void WebrtcVideoStream::Core::CaptureNextFrame() {