0

[Capture] Enable Aura Window Subtree Capture

This patch hides the slow window capturer behind a new feature flag,
features::kAuraWindowSubtreeCapture, and enables aura::Window
capture using SubtreeCaptureId and the FrameSinkVideoCapturer, as part
of implementing go/slow-capturer-removal, now that subtree capture
support has been implemented as part of crbug.com/1143930. The
FrameSinkVideoCaptureDevice and FrameSinkVideoCapturerImpl are modified
to handle window capture as well as screen capture.

This patch builds on CL/2593724 by expanding the SubtreeCaptureId
to include the capture_size, which is based on the size of the
originating aura::Window, and then trimming the surface CopyOutputRequest
with that capture size.

Bug: 958175
Co-Authored-By: Piotr Bialecki <bialpio@chromium.org>
Change-Id: Ib5fc9fe9175e00d716d14a407d834b6cb354542f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2846957
Reviewed-by: Ahmed Fakhry <afakhry@chromium.org>
Reviewed-by: mark a. foltz <mfoltz@chromium.org>
Reviewed-by: kylechar <kylechar@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Reviewed-by: Piotr Bialecki <bialpio@chromium.org>
Commit-Queue: Jordan Bayles <jophba@chromium.org>
Cr-Commit-Position: refs/heads/master@{#886053}
This commit is contained in:
Jordan Bayles
2021-05-24 20:16:40 +00:00
committed by Chromium LUCI CQ
parent f05a71ae77
commit eab523920c
33 changed files with 366 additions and 103 deletions

@ -160,6 +160,10 @@ viz::SubtreeCaptureId RenderSurfaceImpl::SubtreeCaptureId() const {
return OwningEffectNode()->subtree_capture_id;
}
gfx::Size RenderSurfaceImpl::SubtreeSize() const {
return OwningEffectNode()->subtree_size;
}
bool RenderSurfaceImpl::ShouldCacheRenderSurface() const {
return OwningEffectNode()->cache_render_surface;
}
@ -391,6 +395,10 @@ RenderSurfaceImpl::CreateRenderPass() {
pass->backdrop_filter_bounds = BackdropFilterBounds();
pass->generate_mipmap = TrilinearFiltering();
pass->subtree_capture_id = SubtreeCaptureId();
// The subtree size may be slightly larger than our content rect during
// some animations, so we clamp it here.
pass->subtree_size = SubtreeSize();
pass->subtree_size.SetToMin(content_rect().size());
pass->cache_render_pass = ShouldCacheRenderSurface();
pass->has_damage_from_contributing_content =
HasDamageFromeContributingContent();

@ -23,6 +23,7 @@
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "ui/gfx/geometry/rect.h"
#include "ui/gfx/geometry/rect_f.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/mask_filter_info.h"
#include "ui/gfx/transform.h"
@ -177,8 +178,15 @@ class CC_EXPORT RenderSurfaceImpl {
bool HasCopyRequest() const;
// The capture identifier for this render surface and its originating effect
// node. If empty, this surface has not been selected as a subtree capture and
// is either a root surface or will not be rendered separately.
viz::SubtreeCaptureId SubtreeCaptureId() const;
// The size of this surface that should be used for cropping capture. If
// empty, the entire size of this surface should be used for capture.
gfx::Size SubtreeSize() const;
bool ShouldCacheRenderSurface() const;
// Returns true if it's required to copy the output of this surface (i.e. when

@ -56,6 +56,7 @@ bool EffectNode::operator==(const EffectNode& other) const {
screen_space_opacity == other.screen_space_opacity &&
backdrop_filter_quality == other.backdrop_filter_quality &&
subtree_capture_id == other.subtree_capture_id &&
subtree_size == other.subtree_size &&
cache_render_surface == other.cache_render_surface &&
has_copy_request == other.has_copy_request &&
filters == other.filters &&
@ -183,6 +184,7 @@ void EffectNode::AsValueInto(base::trace_event::TracedValue* value) const {
}
value->SetString("blend_mode", SkBlendMode_Name(blend_mode));
value->SetString("subtree_capture_id", subtree_capture_id.ToString());
value->SetString("subtree_size", subtree_size.ToString());
value->SetBoolean("cache_render_surface", cache_render_surface);
value->SetBoolean("has_copy_request", has_copy_request);
value->SetBoolean("double_sided", double_sided);

@ -13,6 +13,7 @@
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/skia/include/core/SkBlendMode.h"
#include "ui/gfx/geometry/point_f.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/geometry/size_f.h"
#include "ui/gfx/mask_filter_info.h"
#include "ui/gfx/rrect_f.h"
@ -96,6 +97,7 @@ struct CC_EXPORT EffectNode {
gfx::Vector2dF surface_contents_scale;
viz::SubtreeCaptureId subtree_capture_id;
gfx::Size subtree_size;
bool cache_render_surface : 1;
bool has_copy_request : 1;

@ -10,6 +10,7 @@
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "base/auto_reset.h"
#include "cc/base/math_util.h"
@ -26,6 +27,7 @@
#include "cc/trees/transform_node.h"
#include "components/viz/common/frame_sinks/copy_output_request.h"
#include "ui/gfx/geometry/point_f.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/geometry/vector2d_conversions.h"
namespace cc {
@ -455,6 +457,23 @@ bool PropertyTreeBuilderContext::AddEffectNodeIfNeeded(
node->opacity = layer->opacity();
node->blend_mode = layer->blend_mode();
node->subtree_capture_id = layer->subtree_capture_id();
// Layers marked with a valid |subtree_capture_id| represent a subsection
// of the tree that should be rendered and copied as a separate render pass.
// Using the layer bounds as the subtree size here allows us to crop out
// undesired sections of the render pass, such as the shadow added by the
// shadow layer.
//
// If it becomes desirable to capture a different sub-rectangle of the render
// pass, a new custom size (or potentially rect) can be plumbed through the
// layer to here.
if (node->subtree_capture_id.is_valid()) {
// Layer bounds are specified in layer space, which excludes device and
// page scale factors. While the page scale can be ignored for subtree
// capture purposes, the device scale must be accounted for here.
node->subtree_size = gfx::ScaleToFlooredSize(
layer->bounds(), layer_tree_host_->device_scale_factor());
}
node->cache_render_surface = layer->cache_render_surface();
node->has_copy_request = layer->HasCopyRequest();
node->filters = layer->filters();

@ -1827,5 +1827,32 @@ TEST_F(PropertyTreeBuilderTest,
kRoundedCorner4Radius * kDeviceScale);
}
TEST_F(PropertyTreeBuilderTest, SubtreeSize) {
constexpr viz::SubtreeCaptureId kCaptureId{42};
auto parent = Layer::Create();
host()->SetRootLayer(parent);
auto child = Layer::Create();
parent->AddChild(child);
child->SetSubtreeCaptureId(kCaptureId);
// Layer has empty bounds.
Commit(1.1f);
EffectNode* node = GetEffectNode(child.get());
EXPECT_EQ((gfx::Size{}), node->subtree_size);
EXPECT_EQ(kCaptureId, node->subtree_capture_id);
// Layer has bounds, scaling is 1.
child->SetBounds(gfx::Size{1280, 720});
Commit(1.0f);
node = GetEffectNode(child.get());
EXPECT_EQ((gfx::Size{1280, 720}), node->subtree_size);
// Layer has bounds, scaling is 2.
Commit(2.0f);
node = GetEffectNode(child.get());
EXPECT_EQ((gfx::Size{2560, 1440}), node->subtree_size);
}
} // namespace
} // namespace cc

@ -89,6 +89,7 @@ void CompositorRenderPass::SetAll(
const cc::FilterOperations& backdrop_filters,
const absl::optional<gfx::RRectF>& backdrop_filter_bounds,
SubtreeCaptureId subtree_capture_id,
gfx::Size subtree_size,
bool has_transparent_background,
bool cache_render_pass,
bool has_damage_from_contributing_content,
@ -103,6 +104,7 @@ void CompositorRenderPass::SetAll(
this->backdrop_filters = backdrop_filters;
this->backdrop_filter_bounds = backdrop_filter_bounds;
this->subtree_capture_id = subtree_capture_id;
this->subtree_size = subtree_size;
this->has_transparent_background = has_transparent_background;
this->cache_render_pass = cache_render_pass;
this->has_damage_from_contributing_content =
@ -224,9 +226,9 @@ std::unique_ptr<CompositorRenderPass> CompositorRenderPass::DeepCopy() const {
quad_list.size());
copy_pass->SetAll(id, output_rect, damage_rect, transform_to_root_target,
filters, backdrop_filters, backdrop_filter_bounds,
subtree_capture_id, has_transparent_background,
cache_render_pass, has_damage_from_contributing_content,
generate_mipmap);
subtree_capture_id, subtree_size,
has_transparent_background, cache_render_pass,
has_damage_from_contributing_content, generate_mipmap);
if (shared_quad_state_list.empty()) {
DCHECK(quad_list.empty());

@ -26,6 +26,7 @@
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "ui/gfx/display_color_spaces.h"
#include "ui/gfx/geometry/rect.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/rrect_f.h"
#include "ui/gfx/transform.h"
@ -70,6 +71,7 @@ class VIZ_COMMON_EXPORT CompositorRenderPass : public RenderPassInternal {
const cc::FilterOperations& backdrop_filters,
const absl::optional<gfx::RRectF>& backdrop_filter_bounds,
SubtreeCaptureId subtree_capture_id,
gfx::Size subtree_size,
bool has_transparent_background,
bool cache_render_pass,
bool has_damage_from_contributing_content,
@ -92,6 +94,12 @@ class VIZ_COMMON_EXPORT CompositorRenderPass : public RenderPassInternal {
// pass, so that it can be captured by a FrameSinkVideoCapturer.
SubtreeCaptureId subtree_capture_id;
// A clip size in pixels indicating what subsection of the |output_rect|
// should be copied when |subtree_capture_id| is valid. Must be smaller or
// equal to |output_rect|. If empty, then the full |output_rect| should be
// copied.
gfx::Size subtree_size;
// For testing functions.
// TODO(vmpstr): See if we can clean these up by moving the tests to use
// AggregatedRenderPasses where appropriate.

@ -139,8 +139,9 @@ TEST(CompositorRenderPassTest, CopyAllShouldBeIdentical) {
auto pass = CompositorRenderPass::Create();
pass->SetAll(id, output_rect, damage_rect, transform_to_root, filters,
backdrop_filters, backdrop_filter_bounds, SubtreeCaptureId{1u},
has_transparent_background, cache_render_pass,
has_damage_from_contributing_content, generate_mipmap);
output_rect.size(), has_transparent_background,
cache_render_pass, has_damage_from_contributing_content,
generate_mipmap);
// Two quads using one shared state.
SharedQuadState* shared_state1 = pass->CreateAndAppendSharedQuadState();
@ -192,12 +193,13 @@ TEST(CompositorRenderPassTest, CopyAllShouldBeIdentical) {
bool contrib_generate_mipmap = false;
auto contrib = CompositorRenderPass::Create();
contrib->SetAll(
contrib_id, contrib_output_rect, contrib_damage_rect,
contrib_transform_to_root, contrib_filters, contrib_backdrop_filters,
contrib_backdrop_filter_bounds, SubtreeCaptureId{2u},
contrib_has_transparent_background, contrib_cache_render_pass,
contrib_has_damage_from_contributing_content, contrib_generate_mipmap);
contrib->SetAll(contrib_id, contrib_output_rect, contrib_damage_rect,
contrib_transform_to_root, contrib_filters,
contrib_backdrop_filters, contrib_backdrop_filter_bounds,
SubtreeCaptureId{2u}, contrib_output_rect.size(),
contrib_has_transparent_background, contrib_cache_render_pass,
contrib_has_damage_from_contributing_content,
contrib_generate_mipmap);
SharedQuadState* contrib_shared_state =
contrib->CreateAndAppendSharedQuadState();
@ -249,8 +251,9 @@ TEST(CompositorRenderPassTest, CopyAllWithCulledQuads) {
auto pass = CompositorRenderPass::Create();
pass->SetAll(id, output_rect, damage_rect, transform_to_root, filters,
backdrop_filters, backdrop_filter_bounds, SubtreeCaptureId(),
has_transparent_background, cache_render_pass,
has_damage_from_contributing_content, generate_mipmap);
output_rect.size(), has_transparent_background,
cache_render_pass, has_damage_from_contributing_content,
generate_mipmap);
// A shared state with a quad.
SharedQuadState* shared_state1 = pass->CreateAndAppendSharedQuadState();

@ -744,11 +744,11 @@ TEST_F(DisplayTest, BackdropFilterTest) {
auto bd_pass = CompositorRenderPass::Create();
cc::FilterOperations backdrop_filters;
backdrop_filters.Append(cc::FilterOperation::CreateBlurFilter(5.0));
bd_pass->SetAll(render_pass_id_generator.GenerateNextId(),
sub_surface_rect, no_damage, gfx::Transform(),
cc::FilterOperations(), backdrop_filters,
gfx::RRectF(gfx::RectF(sub_surface_rect), 0),
SubtreeCaptureId(), false, false, false, false);
bd_pass->SetAll(
render_pass_id_generator.GenerateNextId(), sub_surface_rect,
no_damage, gfx::Transform(), cc::FilterOperations(), backdrop_filters,
gfx::RRectF(gfx::RectF(sub_surface_rect), 0), SubtreeCaptureId(),
sub_surface_rect.size(), false, false, false, false);
pass_list.push_back(std::move(bd_pass));
CompositorFrame frame = CompositorFrameBuilder()

@ -20,6 +20,7 @@
#include "components/power_scheduler/power_mode_voter.h"
#include "components/viz/common/frame_sinks/begin_frame_source.h"
#include "components/viz/common/quads/compositor_frame.h"
#include "components/viz/common/quads/compositor_render_pass.h"
#include "components/viz/common/resources/bitmap_allocation.h"
#include "components/viz/common/surfaces/surface_info.h"
#include "components/viz/service/display/display.h"
@ -896,18 +897,36 @@ void CompositorFrameSinkSupport::OnClientCaptureStopped() {
}
}
gfx::Size CompositorFrameSinkSupport::GetActiveFrameSize() {
if (last_activated_surface_id_.is_valid()) {
Surface* current_surface =
surface_manager_->GetSurfaceForId(last_activated_surface_id_);
DCHECK(current_surface);
if (current_surface->HasActiveFrame()) {
DCHECK(current_surface->GetActiveFrame().size_in_pixels() ==
current_surface->size_in_pixels());
return current_surface->size_in_pixels();
gfx::Size CompositorFrameSinkSupport::GetCopyOutputRequestSize(
SubtreeCaptureId subtree_id) const {
if (!last_activated_surface_id_.is_valid()) {
return {};
}
Surface* current_surface =
surface_manager_->GetSurfaceForId(last_activated_surface_id_);
DCHECK(current_surface);
if (!current_surface->HasActiveFrame()) {
return {};
}
// If a subtree is not specified, use the size of the root (last)
// render pass instead.
const CompositorFrame& frame = current_surface->GetActiveFrame();
if (!subtree_id.is_valid()) {
return frame.size_in_pixels();
}
for (auto& render_pass : frame.render_pass_list) {
if (render_pass->subtree_capture_id == subtree_id) {
return !render_pass->subtree_size.IsEmpty()
? render_pass->subtree_size
: render_pass->output_rect.size();
}
}
return gfx::Size();
// No target exists and no CopyOutputRequest will be added.
return {};
}
void CompositorFrameSinkSupport::RequestCopyOfOutput(

@ -182,9 +182,10 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport
// CapturableFrameSink implementation.
void AttachCaptureClient(CapturableFrameSink::Client* client) override;
void DetachCaptureClient(CapturableFrameSink::Client* client) override;
gfx::Size GetCopyOutputRequestSize(
SubtreeCaptureId subtree_id) const override;
void OnClientCaptureStarted() override;
void OnClientCaptureStopped() override;
gfx::Size GetActiveFrameSize() override;
void RequestCopyOfOutput(
PendingCopyOutputRequest pending_copy_output_request) override;
const CompositorFrameMetadata* GetLastActivatedFrameMetadata() override;

@ -1614,4 +1614,56 @@ TEST_F(CompositorFrameSinkSupportTest, ForceFullFrameToActivateSurface) {
begin_frame_source.TestOnBeginFrame(args_animate_only);
}
TEST_F(CompositorFrameSinkSupportTest, GetCopyOutputRequestSize) {
// No surface with active frame.
EXPECT_EQ((gfx::Size{}),
support_->GetCopyOutputRequestSize(SubtreeCaptureId{}));
// Surface with active frame but no capture identifier.
ResourceId first_frame_ids[] = {ResourceId(1), ResourceId(2), ResourceId(3)};
SubmitCompositorFrameWithResources(first_frame_ids,
base::size(first_frame_ids));
EXPECT_EQ((gfx::Size{20, 20}),
support_->GetCopyOutputRequestSize(SubtreeCaptureId{}));
// Render pass with subtree size.
const SurfaceId surface_id(support_->frame_sink_id(), local_surface_id_);
constexpr SubtreeCaptureId kSubtreeId1(22);
auto frame = CompositorFrameBuilder()
.AddDefaultRenderPass()
.AddDefaultRenderPass()
.SetReferencedSurfaces({SurfaceRange(surface_id)})
.Build();
frame.render_pass_list.front()->subtree_capture_id = kSubtreeId1;
frame.render_pass_list.front()->subtree_size = gfx::Size{13, 37};
support_->SubmitCompositorFrame(local_surface_id_, std::move(frame));
EXPECT_EQ(surface_observer_.last_created_surface_id().local_surface_id(),
local_surface_id_);
EXPECT_EQ((gfx::Size{13, 37}),
support_->GetCopyOutputRequestSize(kSubtreeId1));
// Render pass but no subtree size.
constexpr SubtreeCaptureId kSubtreeId2(7);
auto frame_with_output_size =
CompositorFrameBuilder()
.AddDefaultRenderPass()
.AddDefaultRenderPass()
.SetReferencedSurfaces({SurfaceRange(surface_id)})
.Build();
frame_with_output_size.render_pass_list.front()->subtree_capture_id =
kSubtreeId2;
frame_with_output_size.render_pass_list.front()->output_rect =
gfx::Rect{0, 0, 640, 480};
support_->SubmitCompositorFrame(local_surface_id_,
std::move(frame_with_output_size));
EXPECT_EQ(surface_observer_.last_created_surface_id().local_surface_id(),
local_surface_id_);
EXPECT_EQ((gfx::Size{640, 480}),
support_->GetCopyOutputRequestSize(kSubtreeId2));
}
} // namespace viz

@ -53,14 +53,17 @@ class CapturableFrameSink {
virtual void AttachCaptureClient(Client* client) = 0;
virtual void DetachCaptureClient(Client* client) = 0;
// Returns the size of a render pass, either matching the |subtree_id| if set,
// or the root render pass if not set. Returns an empty size if (1) there is
// no active frame, or (2) |subtree_id| is valid/set and no matching render
// pass could be found.
virtual gfx::Size GetCopyOutputRequestSize(
SubtreeCaptureId subtree_id) const = 0;
// Called when a video capture client starts or stops capturing.
virtual void OnClientCaptureStarted() = 0;
virtual void OnClientCaptureStopped() = 0;
// Returns the currently-active frame size, or an empty size if there is no
// active frame.
virtual gfx::Size GetActiveFrameSize() = 0;
// Issues a request for a copy of the next composited frame whose
// LocalSurfaceId is at least |local_surface_id|. Note that if this id is
// default constructed, then the next surface will provide the copy output

@ -350,7 +350,8 @@ void FrameSinkVideoCapturerImpl::RefreshSoon() {
}
// Detect whether the source size changed before attempting capture.
const gfx::Size& source_size = resolved_target_->GetActiveFrameSize();
const gfx::Size source_size =
resolved_target_->GetCopyOutputRequestSize(request_subtree_id_);
if (source_size.IsEmpty()) {
// If the target's surface size is empty, that indicates it has not yet had
// its first frame composited. Since having content is obviously a
@ -358,12 +359,14 @@ void FrameSinkVideoCapturerImpl::RefreshSoon() {
ScheduleRefreshFrame();
return;
}
if (source_size != oracle_->source_size()) {
oracle_->SetSourceSize(source_size);
InvalidateEntireSource();
if (log_to_webrtc_) {
consumer_->OnLog(
base::StringPrintf("VFC: RefreshSoon() changed active frame size: %s",
base::StringPrintf("FrameSinkVideoCapturerImpl::RefreshSoon() "
"changed active frame size: %s",
source_size.ToString().c_str()));
}
}
@ -384,15 +387,28 @@ void FrameSinkVideoCapturerImpl::OnFrameDamaged(
DCHECK(!expected_display_time.is_null());
DCHECK(resolved_target_);
if (frame_size == oracle_->source_size()) {
InvalidateRect(damage_rect);
const gfx::Size pass_size =
resolved_target_->GetCopyOutputRequestSize(request_subtree_id_);
if (pass_size.IsEmpty()) {
return;
}
if (pass_size == oracle_->source_size()) {
if (request_subtree_id_.is_valid()) {
// The damage_rect may not be in the same coordinate space when we have
// a valid request subtree identifier, so to be safe we just invalidate
// the entire source.
InvalidateEntireSource();
} else {
InvalidateRect(damage_rect);
}
} else {
oracle_->SetSourceSize(frame_size);
oracle_->SetSourceSize(pass_size);
InvalidateEntireSource();
if (log_to_webrtc_ && consumer_) {
consumer_->OnLog(
base::StringPrintf("VFC: OnFramedamaged() changed frame size: %s",
frame_size.ToString().c_str()));
consumer_->OnLog(base::StringPrintf(
"FrameSinkVideoCapturerImpl::OnFrameDamaged() changed frame size: %s",
pass_size.ToString().c_str()));
}
}
@ -634,7 +650,8 @@ void FrameSinkVideoCapturerImpl::MaybeCaptureFrame(
strides = "strides:???";
}
consumer_->OnLog(base::StringPrintf(
"VFC: Ressurecting frame format=%s frame_coded_size: %s "
"FrameSinkVideoCapturerImpl: Resurrecting frame format=%s "
"frame_coded_size: %s "
"frame_visible_rect: %s frame_natural_size: %s %s",
VideoPixelFormatToString(frame->format()).c_str(),
frame->coded_size().ToString().c_str(),
@ -647,16 +664,14 @@ void FrameSinkVideoCapturerImpl::MaybeCaptureFrame(
}
// Request a copy of the next frame from the frame sink.
std::unique_ptr<CopyOutputRequest> request(new CopyOutputRequest(
auto request = std::make_unique<CopyOutputRequest>(
pixel_format_ == media::PIXEL_FORMAT_I420
? CopyOutputRequest::ResultFormat::I420_PLANES
: CopyOutputRequest::ResultFormat::RGBA_BITMAP,
base::BindOnce(&FrameSinkVideoCapturerImpl::DidCopyFrame,
capture_weak_factory_.GetWeakPtr(), capture_frame_number,
oracle_frame_number, content_version_, content_rect,
VideoCaptureOverlay::MakeCombinedRenderer(
GetOverlaysInOrder(), content_rect, frame->format()),
std::move(frame), base::TimeTicks::Now())));
std::move(frame), base::TimeTicks::Now()));
request->set_result_task_runner(base::SequencedTaskRunnerHandle::Get());
request->set_source(copy_request_source_);
request->set_area(gfx::Rect(source_size));
@ -676,7 +691,7 @@ void FrameSinkVideoCapturerImpl::MaybeCaptureFrame(
std::string format =
pixel_format_ == media::PIXEL_FORMAT_I420 ? "I420" : "RGBA_bitmap";
consumer_->OnLog(base::StringPrintf(
"VFC: Sending CopyRequest: "
"FrameSinkVideoCapturerImpl: Sending CopyRequest: "
"format=%s area:%s "
"scale_from: %s "
"scale_to: %s "
@ -695,7 +710,6 @@ void FrameSinkVideoCapturerImpl::DidCopyFrame(
OracleFrameNumber oracle_frame_number,
int64_t content_version,
const gfx::Rect& content_rect,
VideoCaptureOverlay::OnceRenderer overlay_renderer,
scoped_refptr<VideoFrame> frame,
base::TimeTicks request_time,
std::unique_ptr<CopyOutputResult> result) {
@ -727,8 +741,9 @@ void FrameSinkVideoCapturerImpl::DidCopyFrame(
break;
}
consumer_->OnLog(base::StringPrintf(
"VFC: got CopyOutputResult: format=%s size:%s frame_coded_size: %s "
"frame_visible_rect: %s frame_natural_size: %s content_rect: %s %s",
"FrameSinkVideoCapturerImpl: got CopyOutputResult: format=%s size:%s "
"frame_coded_size: %s frame_visible_rect: %s frame_natural_size: %s "
"content_rect: %s %s",
format.c_str(), result->size().ToString().c_str(),
frame->coded_size().ToString().c_str(),
frame->visible_rect().ToString().c_str(),
@ -788,6 +803,8 @@ void FrameSinkVideoCapturerImpl::DidCopyFrame(
}
if (frame) {
auto overlay_renderer = VideoCaptureOverlay::MakeCombinedRenderer(
GetOverlaysInOrder(), content_rect, frame->format());
if (overlay_renderer) {
std::move(overlay_renderer).Run(frame.get());
}
@ -878,9 +895,7 @@ void FrameSinkVideoCapturerImpl::MaybeDeliverFrame(
frame->timestamp().InMicroseconds());
// Clone a handle to the shared memory backing the populated video frame, to
// send to the consumer. The handle is READ_WRITE because the consumer is free
// to modify the content further (so long as it undoes its changes before the
// InFlightFrameDelivery::Done() call).
// send to the consumer.
base::ReadOnlySharedMemoryRegion handle =
frame_pool_.CloneHandleForDelivery(frame.get());
DCHECK(handle.IsValid());

@ -65,7 +65,7 @@ class FrameSinkVideoCapturerManager;
// known to it.
//
// Once the target is resolved, this capturer attaches to it to receive events
// of interest regarding the frame flow, display timiming, and changes to the
// of interest regarding the frame flow, display timing, and changes to the
// frame sink's surface. For some subset of frames, decided by
// media::VideoCaptureOracle, this capturer will make a CopyOutputRequest on the
// surface. Successful CopyOutputResults are then copied into pooled shared
@ -203,7 +203,6 @@ class VIZ_SERVICE_EXPORT FrameSinkVideoCapturerImpl final
OracleFrameNumber oracle_frame_number,
int64_t content_version,
const gfx::Rect& content_rect,
VideoCaptureOverlay::OnceRenderer overlay_renderer,
scoped_refptr<media::VideoFrame> frame,
base::TimeTicks request_time,
std::unique_ptr<CopyOutputResult> result);

@ -249,12 +249,15 @@ class FakeCapturableFrameSink : public CapturableFrameSink {
client_ = nullptr;
}
gfx::Size GetCopyOutputRequestSize(
SubtreeCaptureId subtree_id) const override {
return source_size();
}
void OnClientCaptureStarted() override { ++number_clients_capturing_; }
void OnClientCaptureStopped() override { --number_clients_capturing_; }
gfx::Size GetActiveFrameSize() override { return source_size(); }
void RequestCopyOfOutput(
PendingCopyOutputRequest pending_copy_output_request) override {
auto& request = pending_copy_output_request.copy_output_request;

@ -530,8 +530,8 @@ SurfaceAnimationManager::CopyPassWithoutSharedElementQuads(
source_pass.id, source_pass.output_rect, source_pass.damage_rect,
source_pass.transform_to_root_target, source_pass.filters,
source_pass.backdrop_filters, source_pass.backdrop_filter_bounds,
source_pass.subtree_capture_id, source_pass.has_transparent_background,
source_pass.cache_render_pass,
source_pass.subtree_capture_id, source_pass.subtree_size,
source_pass.has_transparent_background, source_pass.cache_render_pass,
source_pass.has_damage_from_contributing_content,
source_pass.generate_mipmap);

@ -2368,6 +2368,8 @@ source_set("browser") {
]
deps += [ "//ui/base/cursor" ]
}
# TODO(crbug.com/1210549): Slow capturer should be removed once new path is stable.
if (is_chromeos_ash) {
sources += [
"media/capture/slow_capture_overlay_chromeos.cc",

@ -7,6 +7,7 @@
#include <utility>
#include "base/bind.h"
#include "base/feature_list.h"
#include "base/location.h"
#include "base/macros.h"
#include "base/memory/ref_counted.h"
@ -17,6 +18,7 @@
#include "content/public/browser/browser_task_traits.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/desktop_media_id.h"
#include "content/public/common/content_features.h"
#include "media/base/bind_to_current_loop.h"
#include "mojo/public/cpp/bindings/pending_receiver.h"
#include "mojo/public/cpp/bindings/self_owned_receiver.h"
@ -24,9 +26,9 @@
#include "ui/aura/window.h"
#include "ui/aura/window_observer.h"
#include "ui/aura/window_occlusion_tracker.h"
#if BUILDFLAG(IS_CHROMEOS_ASH)
#include "content/browser/media/capture/slow_window_capturer_chromeos.h"
#include "content/public/common/content_features.h"
#endif
namespace content {
@ -80,15 +82,21 @@ class AuraWindowVideoCaptureDevice::WindowTracker final
DCHECK(!target_window_);
target_window_ = DesktopMediaID::GetNativeWindowById(source_id);
if (target_window_ &&
FrameSinkVideoCaptureDevice::VideoCaptureTarget target;
if (target_window_) {
target.frame_sink_id = target_window_->GetRootWindow()->GetFrameSinkId();
#if BUILDFLAG(IS_CHROMEOS_ASH)
// See class comments for SlowWindowCapturerChromeOS.
(source_id.type == DesktopMediaID::TYPE_WINDOW ||
target_window_->GetFrameSinkId().is_valid()) &&
#else
target_window_->GetFrameSinkId().is_valid() &&
if (base::FeatureList::IsEnabled(features::kAuraWindowSubtreeCapture)) {
if (!target_window_->IsRootWindow()) {
capture_request_ = target_window_->MakeWindowCapturable();
target.subtree_capture_id = capture_request_.GetCaptureId();
}
}
#endif
true) {
}
if (target.frame_sink_id.is_valid()) {
target_ = target;
#if BUILDFLAG(IS_CHROMEOS_ASH)
force_visible_.emplace(target_window_);
#endif
@ -96,7 +104,7 @@ class AuraWindowVideoCaptureDevice::WindowTracker final
device_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetChanged, device_,
target_window_->GetFrameSinkId()));
std::move(target)));
// Note: The MouseCursorOverlayController runs on the UI thread. It's also
// important that SetTargetView() be called in the current stack while
// |target_window_| is known to be a valid pointer.
@ -117,6 +125,8 @@ class AuraWindowVideoCaptureDevice::WindowTracker final
target_window_->RemoveObserver(this);
target_window_ = nullptr;
capture_request_ = aura::ScopedWindowCaptureRequest();
#if BUILDFLAG(IS_CHROMEOS_ASH)
force_visible_.reset();
#endif
@ -128,6 +138,31 @@ class AuraWindowVideoCaptureDevice::WindowTracker final
cursor_controller_->SetTargetView(gfx::NativeView());
}
#if BUILDFLAG(IS_CHROMEOS_ASH)
void OnWindowAddedToRootWindow(aura::Window* window) final {
DCHECK_CURRENTLY_ON(BrowserThread::UI);
DCHECK_EQ(window, target_window_);
// The legacy capture path doesn't need to track frame sink ID changes.
if (!base::FeatureList::IsEnabled(features::kAuraWindowSubtreeCapture)) {
return;
}
viz::FrameSinkId new_frame_sink_id =
target_window_->GetRootWindow()->GetFrameSinkId();
// Since the window is not destroyed, only re-parented, we can keep the
// same subtree ID and only update the FrameSinkId of the target.
if (new_frame_sink_id != target_.frame_sink_id) {
target_.frame_sink_id = new_frame_sink_id;
device_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetChanged, device_,
target_));
}
}
#endif
private:
// |device_| may be dereferenced only by tasks run by |device_task_runner_|.
const base::WeakPtr<FrameSinkVideoCaptureDevice> device_;
@ -146,6 +181,9 @@ class AuraWindowVideoCaptureDevice::WindowTracker final
force_visible_;
#endif
aura::ScopedWindowCaptureRequest capture_request_;
FrameSinkVideoCaptureDevice::VideoCaptureTarget target_;
DISALLOW_COPY_AND_ASSIGN(WindowTracker);
};
@ -172,9 +210,11 @@ void AuraWindowVideoCaptureDevice::CreateCapturer(
return;
}
if (tracker->target_type() == DesktopMediaID::TYPE_WINDOW) {
VLOG(1) << "AuraWindowVideoCaptureDevice is using the LAME "
"capturer. :(";
if (tracker->target_type() == DesktopMediaID::TYPE_WINDOW &&
!base::FeatureList::IsEnabled(
features::kAuraWindowSubtreeCapture)) {
VLOG(1) << "AuraWindowVideoCaptureDevice is using the legacy "
"slow capturer.";
mojo::MakeSelfOwnedReceiver(
std::make_unique<SlowWindowCapturerChromeOS>(
tracker->target_window()),

@ -36,9 +36,11 @@ class CONTENT_EXPORT AuraWindowVideoCaptureDevice final
#if BUILDFLAG(IS_CHROMEOS_ASH)
protected:
// Overrides FrameSinkVideoCaptureDevice::CreateCapturer() to create a
// LameWindowCapturerChromeOS for window capture where compositor frame sinks
// are not present. See class comments for LameWindowCapturerChromeOS for
// further details.
// SlowWindowCapturerChromeOS for window capture where compositor frame sinks
// are not present. If the new features::kAuraWindowSubtreeCapture flag is
// enabled, this will use the base's frame sink capture code.
// TODO(crbug.com/1210549): remove once we have determined the new path is
// stable.
void CreateCapturer(
mojo::PendingReceiver<viz::mojom::FrameSinkVideoCapturer> receiver) final;
#endif

@ -120,8 +120,8 @@ void FrameSinkVideoCaptureDevice::AllocateAndStartWithReceiver(
constraints.max_frame_size,
constraints.fixed_aspect_ratio);
if (target_.is_valid()) {
capturer_->ChangeTarget(target_, viz::SubtreeCaptureId());
if (target_.frame_sink_id.is_valid()) {
capturer_->ChangeTarget(target_.frame_sink_id, target_.subtree_capture_id);
}
#if !defined(OS_ANDROID)
@ -298,22 +298,22 @@ void FrameSinkVideoCaptureDevice::OnLog(const std::string& message) {
}
void FrameSinkVideoCaptureDevice::OnTargetChanged(
const viz::FrameSinkId& frame_sink_id) {
const FrameSinkVideoCaptureDevice::VideoCaptureTarget& target) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
target_ = frame_sink_id;
target_ = target;
if (capturer_) {
capturer_->ChangeTarget(target_.is_valid()
? absl::make_optional<viz::FrameSinkId>(target_)
: absl::nullopt,
viz::SubtreeCaptureId());
capturer_->ChangeTarget(
target_.frame_sink_id.is_valid()
? absl::make_optional<viz::FrameSinkId>(target_.frame_sink_id)
: absl::nullopt,
target.subtree_capture_id);
}
}
void FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
OnTargetChanged(viz::FrameSinkId());
OnTargetChanged(VideoCaptureTarget{});
OnFatalError("Capture target has been permanently lost.");
}

@ -86,9 +86,29 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice
void OnStopped() final;
void OnLog(const std::string& message) final;
// All of the information necessary to select a target for capture.
struct VideoCaptureTarget {
// The target frame sink id.
viz::FrameSinkId frame_sink_id;
// The subtree capture identifier--may be default initialized to indicate
// that the entire frame sink (defined by |frame_sink_id|) should be
// captured.
viz::SubtreeCaptureId subtree_capture_id;
inline bool operator==(const VideoCaptureTarget& other) const {
return frame_sink_id == other.frame_sink_id &&
subtree_capture_id == other.subtree_capture_id;
}
inline bool operator!=(const VideoCaptureTarget& other) const {
return !(*this == other);
}
};
// These are called to notify when the capture target has changed or was
// permanently lost.
virtual void OnTargetChanged(const viz::FrameSinkId& frame_sink_id);
virtual void OnTargetChanged(const VideoCaptureTarget& target);
virtual void OnTargetPermanentlyLost();
protected:
@ -138,7 +158,7 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice
// Current capture target. This is cached to resolve a race where
// OnTargetChanged() can be called before the |capturer_| is created in
// OnCapturerCreated().
viz::FrameSinkId target_;
VideoCaptureTarget target_;
// The requested format, rate, and other capture constraints.
media::VideoCaptureParams capture_params_;

@ -104,9 +104,12 @@ class MockFrameSinkVideoCapturer : public viz::mojom::FrameSinkVideoCapturer {
void ChangeTarget(const absl::optional<viz::FrameSinkId>& frame_sink_id,
const viz::SubtreeCaptureId& subtree_capture_id) final {
DCHECK_NOT_ON_DEVICE_THREAD();
MockChangeTarget(frame_sink_id ? *frame_sink_id : viz::FrameSinkId());
MockChangeTarget(FrameSinkVideoCaptureDevice::VideoCaptureTarget{
frame_sink_id.value_or(viz::FrameSinkId{}), subtree_capture_id});
}
MOCK_METHOD1(MockChangeTarget, void(const viz::FrameSinkId& frame_sink_id));
MOCK_METHOD1(
MockChangeTarget,
void(const FrameSinkVideoCaptureDevice::VideoCaptureTarget& target));
void Start(
mojo::PendingRemote<viz::mojom::FrameSinkVideoConsumer> consumer) final {
DCHECK_NOT_ON_DEVICE_THREAD();
@ -334,12 +337,13 @@ class FrameSinkVideoCaptureDeviceTest : public testing::Test {
EXPECT_CALL(capturer_, SetMinCapturePeriod(kMinCapturePeriod));
EXPECT_CALL(capturer_,
SetResolutionConstraints(kResolution, kResolution, _));
constexpr viz::FrameSinkId frame_sink_id(1, 1);
EXPECT_CALL(capturer_, MockChangeTarget(frame_sink_id));
FrameSinkVideoCaptureDevice::VideoCaptureTarget target{
.frame_sink_id = {1, 1}};
EXPECT_CALL(capturer_, MockChangeTarget(target));
EXPECT_CALL(capturer_, MockStart(NotNull()));
EXPECT_FALSE(capturer_.is_bound());
POST_DEVICE_METHOD_CALL(OnTargetChanged, frame_sink_id);
POST_DEVICE_METHOD_CALL(OnTargetChanged, target);
POST_DEVICE_METHOD_CALL(AllocateAndStartWithReceiver, GetCaptureParams(),
std::move(receiver));
WAIT_FOR_DEVICE_TASKS();
@ -578,7 +582,9 @@ TEST_F(FrameSinkVideoCaptureDeviceTest, ShutsDownOnFatalError) {
// consumption, unbind the capturer, log an error with the VideoFrameReceiver,
// and destroy the VideoFrameReceiver.
{
EXPECT_CALL(capturer_, MockChangeTarget(viz::FrameSinkId()));
EXPECT_CALL(
capturer_,
MockChangeTarget(FrameSinkVideoCaptureDevice::VideoCaptureTarget{}));
EXPECT_CALL(capturer_, MockStop());
POST_DEVICE_METHOD_CALL0(OnTargetPermanentlyLost);
WAIT_FOR_DEVICE_TASKS();

@ -54,7 +54,8 @@ class ViewsWidgetVideoCaptureDeviceMac::UIThreadDelegate
device_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetChanged, device_,
scoped_cg_window_id_->GetFrameSinkId()));
FrameSinkVideoCaptureDevice::VideoCaptureTarget{
scoped_cg_window_id_->GetFrameSinkId()}));
} else {
// It is entirely possible (although unlikely) that the window
// corresponding to |cg_window_id| be destroyed between when the capture

@ -234,8 +234,9 @@ void WebContentsFrameTracker::OnPossibleTargetChange() {
target_frame_sink_id_ = frame_sink_id;
device_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebContentsVideoCaptureDevice::OnTargetChanged, device_,
frame_sink_id));
base::BindOnce(
&WebContentsVideoCaptureDevice::OnTargetChanged, device_,
FrameSinkVideoCaptureDevice::VideoCaptureTarget{frame_sink_id}));
}
SetTargetView(web_contents()->GetNativeView());

@ -71,7 +71,8 @@ class MockCaptureDevice : public WebContentsVideoCaptureDevice,
public base::SupportsWeakPtr<MockCaptureDevice> {
public:
using WebContentsVideoCaptureDevice::AsWeakPtr;
MOCK_METHOD1(OnTargetChanged, void(const viz::FrameSinkId&));
MOCK_METHOD1(OnTargetChanged,
void(const FrameSinkVideoCaptureDevice::VideoCaptureTarget&));
MOCK_METHOD0(OnTargetPermanentlyLost, void());
};
@ -313,7 +314,10 @@ TEST_F(WebContentsFrameTrackerTest, NotifiesOfLostTargets) {
// test the observer callbacks here.
TEST_F(WebContentsFrameTrackerTest, NotifiesOfTargetChanges) {
const viz::FrameSinkId kNewId(42, 1337);
EXPECT_CALL(*device(), OnTargetChanged(kNewId)).Times(1);
EXPECT_CALL(
*device(),
OnTargetChanged(FrameSinkVideoCaptureDevice::VideoCaptureTarget{kNewId}))
.Times(1);
SetFrameSinkId(kNewId);
// The tracker doesn't actually use the frame host information, just
// posts a possible target change.

@ -2500,6 +2500,8 @@ test("content_unittests") {
sources +=
[ "../browser/media/capture/desktop_capture_device_unittest.cc" ]
}
# TODO(crbug.com/1210549): Slow capturer should be removed once new path is stable.
if (is_chromeos_ash) {
sources += [
"../browser/media/capture/slow_capture_overlay_chromeos_unittest.cc",

@ -27,6 +27,7 @@ bool StructTraits<viz::mojom::CompositorRenderPassDataView,
!data.ReadBackdropFilters(&(*out)->backdrop_filters) ||
!data.ReadBackdropFilterBounds(&(*out)->backdrop_filter_bounds) ||
!data.ReadSubtreeCaptureId(&(*out)->subtree_capture_id) ||
!data.ReadSubtreeSize(&(*out)->subtree_size) ||
!data.ReadCopyRequests(&(*out)->copy_requests) ||
!data.ReadId(&(*out)->id)) {
return false;
@ -36,6 +37,10 @@ bool StructTraits<viz::mojom::CompositorRenderPassDataView,
viz::SetDeserializationCrashKeyString("Invalid render pass ID");
return false;
}
if ((*out)->subtree_size.width() > (*out)->output_rect.size().width() ||
(*out)->subtree_size.height() > (*out)->output_rect.size().height()) {
return false;
}
(*out)->has_transparent_background = data.has_transparent_background();
(*out)->cache_render_pass = data.cache_render_pass();
(*out)->has_damage_from_contributing_content =

@ -62,9 +62,15 @@ struct StructTraits<viz::mojom::CompositorRenderPassDataView,
static viz::SubtreeCaptureId subtree_capture_id(
const std::unique_ptr<viz::CompositorRenderPass>& input) {
DCHECK_LE(input->subtree_size.width(), input->output_rect.size().width());
DCHECK_LE(input->subtree_size.height(), input->output_rect.size().height());
return input->subtree_capture_id;
}
static gfx::Size subtree_size(
const std::unique_ptr<viz::CompositorRenderPass>& input) {
return input->subtree_size;
}
static bool has_transparent_background(
const std::unique_ptr<viz::CompositorRenderPass>& input) {
return input->has_transparent_background;

@ -208,8 +208,9 @@ class VizSerializationPerfTest : public testing::Test {
auto pass_in = CompositorRenderPass::Create();
pass_in->SetAll(root_id, arbitrary_rect1, arbitrary_rect2,
arbitrary_matrix1, arbitrary_filters2, arbitrary_filters1,
arbitrary_rrectf1, SubtreeCaptureId(), arbitrary_bool1,
arbitrary_bool1, arbitrary_bool1, arbitrary_bool1);
arbitrary_rrectf1, SubtreeCaptureId(),
arbitrary_rect1.size(), arbitrary_bool1, arbitrary_bool1,
arbitrary_bool1, arbitrary_bool1);
// Texture quads
for (uint32_t i = 0; i < 10; ++i) {

@ -712,9 +712,9 @@ TEST_F(StructTraitsTest, RenderPass) {
auto input = CompositorRenderPass::Create();
input->SetAll(render_pass_id, output_rect, damage_rect, transform_to_root,
filters, backdrop_filters, backdrop_filter_bounds,
subtree_capture_id, has_transparent_background,
cache_render_pass, has_damage_from_contributing_content,
generate_mipmap);
subtree_capture_id, output_rect.size(),
has_transparent_background, cache_render_pass,
has_damage_from_contributing_content, generate_mipmap);
input->copy_requests.push_back(CopyOutputRequest::CreateStubForTesting());
const gfx::Rect copy_output_area(24, 42, 75, 57);
input->copy_requests.back()->set_area(copy_output_area);
@ -856,7 +856,7 @@ TEST_F(StructTraitsTest, RenderPassWithEmptySharedQuadStateList) {
auto input = CompositorRenderPass::Create();
input->SetAll(render_pass_id, output_rect, damage_rect, transform_to_root,
cc::FilterOperations(), cc::FilterOperations(),
backdrop_filter_bounds, subtree_capture_id,
backdrop_filter_bounds, subtree_capture_id, output_rect.size(),
has_transparent_background, cache_render_pass,
has_damage_from_contributing_content, generate_mipmap);
@ -875,6 +875,7 @@ TEST_F(StructTraitsTest, RenderPassWithEmptySharedQuadStateList) {
EXPECT_EQ(transform_to_root, output->transform_to_root_target);
EXPECT_EQ(backdrop_filter_bounds, output->backdrop_filter_bounds);
EXPECT_EQ(subtree_capture_id, output->subtree_capture_id);
EXPECT_EQ(output_rect.size(), output->subtree_size);
EXPECT_FALSE(output->subtree_capture_id.is_valid());
EXPECT_EQ(has_transparent_background, output->has_transparent_background);
}

@ -23,6 +23,7 @@ struct CompositorRenderPass {
FilterOperations backdrop_filters;
gfx.mojom.RRectF? backdrop_filter_bounds;
SubtreeCaptureId subtree_capture_id;
gfx.mojom.Size subtree_size;
bool has_transparent_background;
bool cache_render_pass = false;
bool has_damage_from_contributing_content = false;