0

Video capture with GpuMemoryBuffer - VideoCaptureImpl and service

This CL allows the GpuMemoryBuffer video capture buffers to be
broadcasted to and handled by the renderer process.

The GMB buffers are bound to GPU textures on the renderer process and
the DMA-buf fds are wrapped in GpuMemoryBuffer VideoFrames.  The GMB
VideoFrames are associated with the bound GPU textures and sent to the
clients.

Some methods in gpu::GpuMemoryBufferSupport are made virtual to
facilitate unit testing.

Design doc: go/cros-camera:dd:zero-copy

Bug: 982201
Test: blink_platform_unittests
Change-Id: I754a7373edbf649452248d695f53f0c45faf6c99
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1692153
Commit-Queue: Ricky Liang <jcliang@chromium.org>
Reviewed-by: Ken Buchanan <kenrb@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Reviewed-by: Christian Fremerey <chfremer@chromium.org>
Reviewed-by: Daniele Castagna <dcastagna@chromium.org>
Cr-Commit-Position: refs/heads/master@{#698319}
This commit is contained in:
Ricky Liang
2019-09-20 02:43:14 +00:00
committed by Commit Bot
parent 8f920dd7a9
commit 04d3a44f88
15 changed files with 607 additions and 23 deletions

@ -243,6 +243,9 @@ VideoCaptureController::BufferContext::CloneBufferHandle() {
DCHECK(result->get_read_only_shmem_region().IsValid());
} else if (buffer_handle_->is_mailbox_handles()) {
result->set_mailbox_handles(buffer_handle_->get_mailbox_handles()->Clone());
} else if (buffer_handle_->is_gpu_memory_buffer_handle()) {
result->set_gpu_memory_buffer_handle(
buffer_handle_->get_gpu_memory_buffer_handle().Clone());
} else {
NOTREACHED() << "Unexpected video buffer handle type";
}

@ -28,7 +28,7 @@ namespace gpu {
class GPU_EXPORT GpuMemoryBufferSupport {
public:
GpuMemoryBufferSupport();
~GpuMemoryBufferSupport();
virtual ~GpuMemoryBufferSupport();
// Returns the native GPU memory buffer factory type. Returns EMPTY_BUFFER
// type if native buffers are not supported.
@ -53,7 +53,8 @@ class GPU_EXPORT GpuMemoryBufferSupport {
// should match what was used to allocate the |handle|. |callback|, if
// non-null, is called when instance is deleted, which is not necessarily on
// the same thread as this function was called on and instance was created on.
std::unique_ptr<GpuMemoryBufferImpl> CreateGpuMemoryBufferImplFromHandle(
virtual std::unique_ptr<GpuMemoryBufferImpl>
CreateGpuMemoryBufferImplFromHandle(
gfx::GpuMemoryBufferHandle handle,
const gfx::Size& size,
gfx::BufferFormat format,

@ -57,6 +57,11 @@ void CloneSharedBufferToRawFileDescriptorHandle(
#endif
}
void CloneGpuMemoryBufferHandle(const gfx::GpuMemoryBufferHandle& source,
media::mojom::VideoBufferHandlePtr* target) {
(*target)->set_gpu_memory_buffer_handle(source.Clone());
}
} // anonymous namespace
BroadcastingReceiver::ClientContext::ClientContext(
@ -167,8 +172,8 @@ BroadcastingReceiver::BufferContext::CloneBufferHandle(
}
break;
case media::VideoCaptureBufferType::kGpuMemoryBuffer:
// TODO(jcliang): Implement this.
NOTREACHED() << "Unexpected video buffer handle type";
CloneGpuMemoryBufferHandle(buffer_handle_->get_gpu_memory_buffer_handle(),
&result);
break;
}
return result;

@ -1489,6 +1489,7 @@ jumbo_component("platform") {
"//ui/base:base",
"//ui/gfx",
"//ui/gfx/geometry",
"//ui/gfx/mojom",
]
if (is_mac) {
@ -1650,6 +1651,7 @@ jumbo_static_library("test_support") {
deps = [
"//base/test:test_support",
"//media:test_support",
"//mojo/core/embedder",
"//mojo/public/cpp/bindings",
"//services/service_manager/public/cpp",
@ -1848,6 +1850,8 @@ jumbo_source_set("blink_platform_unittests_sources") {
"transforms/rotation_test.cc",
"transforms/transform_operations_test.cc",
"transforms/transformation_matrix_test.cc",
"video_capture/gpu_memory_buffer_test_support.cc",
"video_capture/gpu_memory_buffer_test_support.h",
"video_capture/video_capture_impl_test.cc",
"weborigin/known_ports_test.cc",
"weborigin/kurl_test.cc",

@ -16,7 +16,7 @@
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/public/platform/modules/video_capture/web_video_capture_impl_manager.h"
#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
#include "third_party/blink/renderer/platform/video_capture/gpu_memory_buffer_test_support.h"
#include "third_party/blink/renderer/platform/video_capture/video_capture_impl.h"
using media::BindToCurrentLoop;
@ -223,7 +223,8 @@ class VideoCaptureImplManagerTest : public ::testing::Test,
}
base::test::TaskEnvironment task_environment_;
ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_;
ScopedTestingPlatformSupport<TestingPlatformSupportForGpuMemoryBuffer>
platform_;
base::RunLoop cleanup_run_loop_;
std::unique_ptr<MockVideoCaptureImplManager> manager_;

@ -30,6 +30,7 @@ typemaps = [
"//third_party/blink/public/common/mediastream/media_stream.typemap",
"//third_party/blink/public/common/screen_orientation/screen_orientation_lock_types.typemap",
"//third_party/blink/public/mojom/bluetooth/bluetooth.typemap",
"//ui/gfx/mojom/buffer_types_for_blink.typemap",
"//ui/gfx/mojom/color_space_for_blink.typemap",
"//ui/gfx/mojom/transform.typemap",
]

@ -10,10 +10,25 @@ include_rules = [
"+media/capture",
"+third_party/blink/renderer/platform/wtf",
"+third_party/blink/renderer/platform/platform_export.h",
# Dependencies for GpuMemoryBuffer.
"+gpu/command_buffer/client/shared_image_interface.h",
"+gpu/command_buffer/common/shared_image_usage.h",
"+gpu/ipc/common/gpu_memory_buffer_support.h",
"+media/video/gpu_video_accelerator_factories.h",
]
specific_include_rules = {
"video_capture_impl_test.cc": [
"+third_party/blink/renderer/platform/testing/gpu_memory_buffer_test_support.h",
],
"gpu_memory_buffer_test_support.cc": [
"+components/viz/test/test_context_provider.h",
"+media/video/fake_gpu_memory_buffer.h",
"+media/video/mock_gpu_video_accelerator_factories.h",
],
"gpu_memory_buffer_test_support.h": [
"+gpu/ipc/common/gpu_memory_buffer_support.h",
"+third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h",
],
}

@ -0,0 +1,79 @@
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/blink/renderer/platform/video_capture/gpu_memory_buffer_test_support.h"
#include "components/viz/test/test_context_provider.h"
#include "media/video/fake_gpu_memory_buffer.h"
#include "media/video/mock_gpu_video_accelerator_factories.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
using ::testing::Return;
namespace blink {
namespace {
class FakeGpuMemoryBufferImpl : public gpu::GpuMemoryBufferImpl {
public:
FakeGpuMemoryBufferImpl(const gfx::Size& size, gfx::BufferFormat format)
: gpu::GpuMemoryBufferImpl(
gfx::GpuMemoryBufferId(),
size,
format,
gpu::GpuMemoryBufferImpl::DestructionCallback()),
fake_gmb_(std::make_unique<media::FakeGpuMemoryBuffer>(size, format)) {}
// gfx::GpuMemoryBuffer implementation
bool Map() override { return fake_gmb_->Map(); }
void* memory(size_t plane) override { return fake_gmb_->memory(plane); }
void Unmap() override { fake_gmb_->Unmap(); }
int stride(size_t plane) const override { return fake_gmb_->stride(plane); }
gfx::GpuMemoryBufferType GetType() const override {
return fake_gmb_->GetType();
}
gfx::GpuMemoryBufferHandle CloneHandle() const override {
return fake_gmb_->CloneHandle();
}
private:
std::unique_ptr<media::FakeGpuMemoryBuffer> fake_gmb_;
};
} // namespace
std::unique_ptr<gpu::GpuMemoryBufferImpl>
FakeGpuMemoryBufferSupport::CreateGpuMemoryBufferImplFromHandle(
gfx::GpuMemoryBufferHandle handle,
const gfx::Size& size,
gfx::BufferFormat format,
gfx::BufferUsage usage,
gpu::GpuMemoryBufferImpl::DestructionCallback callback) {
return std::make_unique<FakeGpuMemoryBufferImpl>(size, format);
}
TestingPlatformSupportForGpuMemoryBuffer::
TestingPlatformSupportForGpuMemoryBuffer()
: sii_(new viz::TestSharedImageInterface),
gpu_factories_(new media::MockGpuVideoAcceleratorFactories(sii_.get())),
media_thread_("TestingMediaThread") {
gpu_factories_->SetVideoFrameOutputFormat(
media::GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB);
media_thread_.Start();
ON_CALL(*gpu_factories_, GetTaskRunner())
.WillByDefault(Return(media_thread_.task_runner()));
}
TestingPlatformSupportForGpuMemoryBuffer::
~TestingPlatformSupportForGpuMemoryBuffer() {
media_thread_.Stop();
}
media::GpuVideoAcceleratorFactories*
TestingPlatformSupportForGpuMemoryBuffer::GetGpuFactories() {
return gpu_factories_.get();
}
} // namespace blink

@ -0,0 +1,48 @@
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_VIDEO_CAPTURE_GPU_MEMORY_BUFFER_TEST_SUPPORT_H_
#define THIRD_PARTY_BLINK_RENDERER_PLATFORM_VIDEO_CAPTURE_GPU_MEMORY_BUFFER_TEST_SUPPORT_H_
#include <memory>
#include "gpu/ipc/common/gpu_memory_buffer_support.h"
#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
namespace media {
class MockGpuVideoAcceleratorFactories;
} // namespace media
namespace viz {
class TestSharedImageInterface;
} // namespace viz
namespace blink {
class FakeGpuMemoryBufferSupport : public gpu::GpuMemoryBufferSupport {
public:
std::unique_ptr<gpu::GpuMemoryBufferImpl> CreateGpuMemoryBufferImplFromHandle(
gfx::GpuMemoryBufferHandle handle,
const gfx::Size& size,
gfx::BufferFormat format,
gfx::BufferUsage usage,
gpu::GpuMemoryBufferImpl::DestructionCallback callback) override;
};
class TestingPlatformSupportForGpuMemoryBuffer
: public IOTaskRunnerTestingPlatformSupport {
public:
TestingPlatformSupportForGpuMemoryBuffer();
~TestingPlatformSupportForGpuMemoryBuffer() override;
media::GpuVideoAcceleratorFactories* GetGpuFactories() override;
private:
std::unique_ptr<viz::TestSharedImageInterface> sii_;
std::unique_ptr<media::MockGpuVideoAcceleratorFactories> gpu_factories_;
base::Thread media_thread_;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_VIDEO_CAPTURE_GPU_MEMORY_BUFFER_TEST_SUPPORT_H_

@ -20,10 +20,14 @@
#include "base/macros.h"
#include "base/stl_util.h"
#include "base/trace_event/trace_event.h"
#include "gpu/command_buffer/client/shared_image_interface.h"
#include "gpu/command_buffer/common/shared_image_usage.h"
#include "gpu/ipc/common/gpu_memory_buffer_support.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/limits.h"
#include "media/base/video_frame.h"
#include "media/capture/mojom/video_capture_types.mojom-blink.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "mojo/public/cpp/system/platform_handle.h"
#include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h"
#include "third_party/blink/public/platform/platform.h"
@ -34,12 +38,32 @@ namespace blink {
using VideoFrameBufferHandleType = media::mojom::blink::VideoBufferHandle::Tag;
// A collection of all types of handles that we use to reference a camera buffer
// backed with GpuMemoryBuffer.
struct GpuMemoryBufferResources {
explicit GpuMemoryBufferResources(gfx::GpuMemoryBufferHandle handle)
: gpu_memory_buffer_handle(std::move(handle)) {}
// Stores the GpuMemoryBufferHandle when a new buffer is first registered.
// |gpu_memory_buffer_handle| is converted to |gpu_memory_buffer| below when
// the camera frame is ready for the first time.
gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle;
// The GpuMemoryBuffer backing the camera frame.
std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer;
// The SharedImage created from |gpu_memory_buffer|.
gpu::Mailbox mailbox;
// The release sync token for |mailbox|.
gpu::SyncToken release_sync_token;
};
struct VideoCaptureImpl::BufferContext
: public base::RefCountedThreadSafe<BufferContext> {
public:
explicit BufferContext(
media::mojom::blink::VideoBufferHandlePtr buffer_handle)
: buffer_type_(buffer_handle->which()) {
BufferContext(media::mojom::blink::VideoBufferHandlePtr buffer_handle,
media::GpuVideoAcceleratorFactories* gpu_factories,
scoped_refptr<base::SingleThreadTaskRunner> media_task_runner)
: buffer_type_(buffer_handle->which()),
gpu_factories_(gpu_factories),
media_task_runner_(std::move(media_task_runner)) {
switch (buffer_type_) {
case VideoFrameBufferHandleType::SHARED_BUFFER_HANDLE:
InitializeFromSharedMemory(
@ -56,8 +80,10 @@ struct VideoCaptureImpl::BufferContext
InitializeFromMailbox(std::move(buffer_handle->get_mailbox_handles()));
break;
case VideoFrameBufferHandleType::GPU_MEMORY_BUFFER_HANDLE:
// TODO(jcliang): Implement this.
NOTREACHED();
CHECK(gpu_factories_);
CHECK(media_task_runner_);
InitializeFromGpuMemoryBufferHandle(
std::move(buffer_handle->get_gpu_memory_buffer_handle()));
break;
}
}
@ -69,6 +95,97 @@ struct VideoCaptureImpl::BufferContext
return mailbox_holders_;
}
gfx::GpuMemoryBufferHandle TakeGpuMemoryBufferHandle() {
return std::move(gmb_resources_->gpu_memory_buffer_handle);
}
void SetGpuMemoryBuffer(
std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer) {
gmb_resources_->gpu_memory_buffer = std::move(gpu_memory_buffer);
}
gfx::GpuMemoryBuffer* GetGpuMemoryBuffer() {
return gmb_resources_->gpu_memory_buffer.get();
}
// Creates SharedImage mailboxes for |gpu_memory_buffer_handle_| and wraps the
// mailboxes with the buffer handles in a DMA-buf VideoFrame. The consumer of
// the VideoFrame can access the data either through mailboxes (e.g. display)
// or through the DMA-buf FDs (e.g. video encoder).
static void BindBufferToTextureOnMediaThread(
scoped_refptr<BufferContext> buffer_context,
media::mojom::blink::VideoFrameInfoPtr info,
std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer,
scoped_refptr<media::VideoFrame> frame,
base::OnceCallback<void(media::mojom::blink::VideoFrameInfoPtr,
scoped_refptr<media::VideoFrame>,
scoped_refptr<BufferContext>)> on_texture_bound) {
DCHECK(buffer_context->media_task_runner_->BelongsToCurrentThread());
DCHECK(buffer_context->gpu_factories_);
DCHECK_EQ(info->pixel_format, media::PIXEL_FORMAT_NV12);
DCHECK_EQ(
buffer_context->gpu_factories_->VideoFrameOutputFormat(
info->pixel_format),
media::GpuVideoAcceleratorFactories::OutputFormat::NV12_SINGLE_GMB);
// Create GPU texture and bind GpuMemoryBuffer to the texture.
auto* sii = buffer_context->gpu_factories_->SharedImageInterface();
unsigned texture_target =
buffer_context->gpu_factories_->ImageTextureTarget(
gpu_memory_buffer->GetFormat());
if (buffer_context->gmb_resources_->mailbox.IsZero()) {
uint32_t usage =
gpu::SHARED_IMAGE_USAGE_GLES2 | gpu::SHARED_IMAGE_USAGE_RASTER |
gpu::SHARED_IMAGE_USAGE_DISPLAY | gpu::SHARED_IMAGE_USAGE_SCANOUT;
buffer_context->gmb_resources_->mailbox = sii->CreateSharedImage(
gpu_memory_buffer.get(),
buffer_context->gpu_factories_->GpuMemoryBufferManager(),
*(info->color_space), usage);
} else {
sii->UpdateSharedImage(buffer_context->gmb_resources_->release_sync_token,
buffer_context->gmb_resources_->mailbox);
}
gpu::SyncToken sync_token = sii->GenUnverifiedSyncToken();
CHECK(!buffer_context->gmb_resources_->mailbox.IsZero());
gpu::MailboxHolder mailbox_holder_array[media::VideoFrame::kMaxPlanes];
mailbox_holder_array[0] = gpu::MailboxHolder(
buffer_context->gmb_resources_->mailbox, sync_token, texture_target);
const auto gmb_size = gpu_memory_buffer->GetSize();
frame = media::VideoFrame::WrapExternalGpuMemoryBuffer(
gfx::Rect(info->visible_rect), gmb_size, std::move(gpu_memory_buffer),
mailbox_holder_array,
base::BindOnce(&BufferContext::MailboxHolderReleased, buffer_context),
info->timestamp);
frame->metadata()->SetBoolean(media::VideoFrameMetadata::ALLOW_OVERLAY,
true);
frame->metadata()->SetBoolean(
media::VideoFrameMetadata::READ_LOCK_FENCES_ENABLED, true);
std::move(on_texture_bound)
.Run(std::move(info), std::move(frame), std::move(buffer_context));
}
static void MailboxHolderReleased(scoped_refptr<BufferContext> buffer_context,
const gpu::SyncToken& release_sync_token) {
if (!buffer_context->media_task_runner_->BelongsToCurrentThread()) {
buffer_context->media_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&BufferContext::MailboxHolderReleased,
std::move(buffer_context), release_sync_token));
return;
}
buffer_context->gmb_resources_->release_sync_token = release_sync_token;
}
static void DestroyTextureOnMediaThread(
media::GpuVideoAcceleratorFactories* gpu_factories,
gpu::Mailbox mailbox,
gpu::SyncToken release_sync_token) {
if (!mailbox.IsZero()) {
auto* sii = gpu_factories->SharedImageInterface();
sii->DestroySharedImage(release_sync_token, mailbox);
}
}
private:
void InitializeFromSharedMemory(mojo::ScopedSharedBufferHandle handle) {
DCHECK(handle.is_valid());
@ -103,8 +220,21 @@ struct VideoCaptureImpl::BufferContext
mailbox_holders_ = std::move(mailbox_handles->mailbox_holder);
}
void InitializeFromGpuMemoryBufferHandle(
gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle) {
gmb_resources_ = std::make_unique<GpuMemoryBufferResources>(
std::move(gpu_memory_buffer_handle));
}
friend class base::RefCountedThreadSafe<BufferContext>;
virtual ~BufferContext() {}
virtual ~BufferContext() {
if (buffer_type_ == VideoFrameBufferHandleType::GPU_MEMORY_BUFFER_HANDLE) {
media_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&BufferContext::DestroyTextureOnMediaThread,
gpu_factories_, gmb_resources_->mailbox,
gmb_resources_->release_sync_token));
}
}
VideoFrameBufferHandleType buffer_type_;
@ -122,6 +252,15 @@ struct VideoCaptureImpl::BufferContext
// Only valid for |buffer_type_ == MAILBOX_HANDLES|.
Vector<gpu::MailboxHolder> mailbox_holders_;
// The following is for |buffer_type == GPU_MEMORY_BUFFER_HANDLE|.
// Uses to create SharedImage from |gpu_memory_buffer_|.
media::GpuVideoAcceleratorFactories* gpu_factories_;
// The task runner that |gpu_factories_| runs on.
const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
std::unique_ptr<GpuMemoryBufferResources> gmb_resources_;
DISALLOW_COPY_AND_ASSIGN(BufferContext);
};
@ -144,12 +283,18 @@ VideoCaptureImpl::VideoCaptureImpl(media::VideoCaptureSessionId session_id)
: device_id_(session_id),
session_id_(session_id),
video_capture_host_for_testing_(nullptr),
state_(blink::VIDEO_CAPTURE_STATE_STOPPED) {
state_(blink::VIDEO_CAPTURE_STATE_STOPPED),
gpu_memory_buffer_support_(new gpu::GpuMemoryBufferSupport()) {
CHECK(!session_id.is_empty());
DETACH_FROM_THREAD(io_thread_checker_);
Platform::Current()->GetBrowserInterfaceBrokerProxy()->GetInterface(
pending_video_capture_host_.InitWithNewPipeAndPassReceiver());
gpu_factories_ = Platform::Current()->GetGpuFactories();
if (gpu_factories_) {
media_task_runner_ = gpu_factories_->GetTaskRunner();
}
}
VideoCaptureImpl::~VideoCaptureImpl() {
@ -272,6 +417,11 @@ void VideoCaptureImpl::OnLog(const String& message) {
GetVideoCaptureHost()->OnLog(device_id_, message);
}
void VideoCaptureImpl::SetGpuMemoryBufferSupportForTesting(
std::unique_ptr<gpu::GpuMemoryBufferSupport> gpu_memory_buffer_support) {
gpu_memory_buffer_support_ = std::move(gpu_memory_buffer_support);
}
void VideoCaptureImpl::OnStateChanged(media::mojom::VideoCaptureState state) {
DVLOG(1) << __func__ << " state: " << state;
DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
@ -331,7 +481,9 @@ void VideoCaptureImpl::OnNewBuffer(
const bool inserted =
client_buffers_
.emplace(buffer_id, new BufferContext(std::move(buffer_handle)))
.emplace(buffer_id,
new BufferContext(std::move(buffer_handle), gpu_factories_,
media_task_runner_))
.second;
DCHECK(inserted);
}
@ -446,11 +598,62 @@ void VideoCaptureImpl::OnBufferReady(
info->timestamp);
break;
}
case VideoFrameBufferHandleType::GPU_MEMORY_BUFFER_HANDLE:
// TODO(jcliang): Implement this.
NOTREACHED();
break;
case VideoFrameBufferHandleType::GPU_MEMORY_BUFFER_HANDLE: {
// Create GpuMemoryBuffer from handle.
if (!buffer_context->GetGpuMemoryBuffer()) {
gfx::BufferFormat gfx_format;
switch (info->pixel_format) {
case media::VideoPixelFormat::PIXEL_FORMAT_NV12:
gfx_format = gfx::BufferFormat::YUV_420_BIPLANAR;
break;
default:
LOG(FATAL) << "Unsupported pixel format";
return;
}
// The GpuMemoryBuffer is allocated and owned by the video capture
// buffer pool from the video capture service process, so we don't need
// to destroy the GpuMemoryBuffer here.
auto gmb =
gpu_memory_buffer_support_->CreateGpuMemoryBufferImplFromHandle(
buffer_context->TakeGpuMemoryBufferHandle(),
gfx::Size(info->coded_size), gfx_format,
gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, base::DoNothing());
buffer_context->SetGpuMemoryBuffer(std::move(gmb));
}
CHECK(buffer_context->GetGpuMemoryBuffer());
// Clone the GpuMemoryBuffer and wrap it in a VideoFrame.
std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
gpu_memory_buffer_support_->CreateGpuMemoryBufferImplFromHandle(
buffer_context->GetGpuMemoryBuffer()->CloneHandle(),
buffer_context->GetGpuMemoryBuffer()->GetSize(),
buffer_context->GetGpuMemoryBuffer()->GetFormat(),
gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, base::DoNothing());
media_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(
&BufferContext::BindBufferToTextureOnMediaThread,
std::move(buffer_context), base::Passed(&info),
base::Passed(&gmb), frame,
media::BindToCurrentLoop(base::BindOnce(
&VideoCaptureImpl::OnVideoFrameReady,
weak_factory_.GetWeakPtr(), buffer_id, reference_time))));
return;
}
}
OnVideoFrameReady(buffer_id, reference_time, std::move(info),
std::move(frame), std::move(buffer_context));
}
void VideoCaptureImpl::OnVideoFrameReady(
int32_t buffer_id,
base::TimeTicks reference_time,
media::mojom::blink::VideoFrameInfoPtr info,
scoped_refptr<media::VideoFrame> frame,
scoped_refptr<BufferContext> buffer_context) {
DCHECK_CALLED_ON_VALID_THREAD(io_thread_checker_);
if (!frame) {
OnFrameDropped(media::VideoCaptureFrameDropReason::
kVideoCaptureImplFailedToWrapDataAsMediaVideoFrame);

@ -20,6 +20,14 @@
#include "third_party/blink/public/common/media/video_capture.h"
#include "third_party/blink/renderer/platform/platform_export.h"
namespace gpu {
class GpuMemoryBufferSupport;
} // namespace gpu
namespace media {
class GpuVideoAcceleratorFactories;
} // namespace media
namespace blink {
// VideoCaptureImpl represents a capture device in renderer process. It provides
@ -73,6 +81,8 @@ class PLATFORM_EXPORT VideoCaptureImpl
media::mojom::blink::VideoCaptureHost* service) {
video_capture_host_for_testing_ = service;
}
void SetGpuMemoryBufferSupportForTesting(
std::unique_ptr<gpu::GpuMemoryBufferSupport> gpu_memory_buffer_support);
// media::mojom::VideoCaptureObserver implementation.
void OnStateChanged(media::mojom::VideoCaptureState state) override;
@ -97,6 +107,12 @@ class PLATFORM_EXPORT VideoCaptureImpl
using BufferFinishedCallback =
base::OnceCallback<void(double consumer_resource_utilization)>;
void OnVideoFrameReady(int32_t buffer_id,
base::TimeTicks reference_time,
media::mojom::blink::VideoFrameInfoPtr info,
scoped_refptr<media::VideoFrame> frame,
scoped_refptr<BufferContext> buffer_context);
void OnAllClientsFinishedConsumingFrame(
int buffer_id,
scoped_refptr<BufferContext> buffer_context,
@ -158,6 +174,12 @@ class PLATFORM_EXPORT VideoCaptureImpl
VideoCaptureState state_;
// Methods of |gpu_factories_| need to run on |media_task_runner_|.
media::GpuVideoAcceleratorFactories* gpu_factories_;
scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
std::unique_ptr<gpu::GpuMemoryBufferSupport> gpu_memory_buffer_support_;
THREAD_CHECKER(io_thread_checker_);
// WeakPtrFactory pointing back to |this| object, for use with

@ -10,6 +10,7 @@
#include "base/macros.h"
#include "base/memory/read_only_shared_memory_region.h"
#include "base/memory/unsafe_shared_memory_region.h"
#include "base/test/bind_test_util.h"
#include "base/test/task_environment.h"
#include "media/capture/mojom/video_capture.mojom-blink.h"
#include "media/capture/mojom/video_capture_types.mojom-blink.h"
@ -17,7 +18,7 @@
#include "mojo/public/cpp/system/platform_handle.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h"
#include "third_party/blink/renderer/platform/video_capture/gpu_memory_buffer_test_support.h"
#include "third_party/blink/renderer/platform/video_capture/video_capture_impl.h"
#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
@ -25,6 +26,7 @@ using ::testing::_;
using ::testing::InSequence;
using ::testing::Invoke;
using ::testing::InvokeWithoutArgs;
using ::testing::Mock;
using ::testing::SaveArg;
using ::testing::WithArgs;
@ -130,6 +132,9 @@ class VideoCaptureImplTest : public ::testing::Test {
video_capture_impl_->OnStateChanged(
media::mojom::VideoCaptureState::STARTED);
}));
video_capture_impl_->SetGpuMemoryBufferSupportForTesting(
std::make_unique<FakeGpuMemoryBufferSupport>());
}
protected:
@ -172,7 +177,18 @@ class VideoCaptureImplTest : public ::testing::Test {
std::move(region)));
}
void SimulateBufferReceived(int buffer_id, const gfx::Size& size) {
void SimulateGpuMemoryBufferCreated(int buffer_id,
gfx::GpuMemoryBufferHandle gmb_handle) {
video_capture_impl_->OnNewBuffer(
buffer_id,
media::mojom::blink::VideoBufferHandle::NewGpuMemoryBufferHandle(
std::move(gmb_handle)));
}
void SimulateBufferReceived(
int buffer_id,
const gfx::Size& size,
media::VideoPixelFormat pixel_format = media::PIXEL_FORMAT_I420) {
media::mojom::blink::VideoFrameInfoPtr info =
media::mojom::blink::VideoFrameInfo::New();
@ -182,9 +198,10 @@ class VideoCaptureImplTest : public ::testing::Test {
info->metadata = frame_metadata.GetInternalValues().Clone();
info->timestamp = now - base::TimeTicks();
info->pixel_format = media::PIXEL_FORMAT_I420;
info->pixel_format = pixel_format;
info->coded_size = WebSize(size);
info->visible_rect = WebRect(gfx::Rect(size));
info->color_space = gfx::ColorSpace();
video_capture_impl_->OnBufferReady(buffer_id, std::move(info));
}
@ -213,8 +230,9 @@ class VideoCaptureImplTest : public ::testing::Test {
const base::UnguessableToken session_id_ = base::UnguessableToken::Create();
base::test::TaskEnvironment task_environment_;
ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_;
const std::unique_ptr<VideoCaptureImpl> video_capture_impl_;
ScopedTestingPlatformSupport<TestingPlatformSupportForGpuMemoryBuffer>
platform_;
std::unique_ptr<VideoCaptureImpl> video_capture_impl_;
MockMojoVideoCaptureHost mock_video_capture_host_;
media::VideoCaptureParams params_small_;
media::VideoCaptureParams params_large_;
@ -358,6 +376,74 @@ TEST_F(VideoCaptureImplTest, BufferReceived_ReadOnlyShmemRegion) {
EXPECT_EQ(mock_video_capture_host_.released_buffer_count(), 0);
}
TEST_F(VideoCaptureImplTest, BufferReceived_GpuMemoryBufferHandle) {
const int kArbitraryBufferId = 11;
// With GpuMemoryBufferHandle, the buffer handle is received on the IO thread
// and passed to a media thread to create a SharedImage. After the SharedImage
// is created and wrapped in a video frame, we pass the video frame back to
// the IO thread to pass to the clients by calling their frame-ready
// callbacks.
base::Thread testing_io_thread("TestingIOThread");
base::WaitableEvent frame_ready_event;
scoped_refptr<media::VideoFrame> frame;
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STOPPED));
EXPECT_CALL(*this, OnFrameReady(_, _))
.WillOnce(
Invoke([&](scoped_refptr<media::VideoFrame> f, base::TimeTicks t) {
// Hold on a reference to the video frame to emulate that we're
// actively using the buffer.
frame = f;
frame_ready_event.Signal();
}));
EXPECT_CALL(mock_video_capture_host_, DoStart(_, session_id_, params_small_));
EXPECT_CALL(mock_video_capture_host_, Stop(_));
EXPECT_CALL(mock_video_capture_host_, ReleaseBuffer(_, kArbitraryBufferId, _))
.Times(0);
// The first half of the test: Create and queue the GpuMemoryBufferHandle.
// VideoCaptureImpl would:
// 1. create a GpuMemoryBuffer out of the handle on |testing_io_thread|
// 2. create a SharedImage from the GpuMemoryBuffer on |media_thread_|
// 3. invoke OnFrameReady callback on |testing_io_thread|
auto create_and_queue_buffer = [&]() {
gfx::GpuMemoryBufferHandle gmb_handle;
gmb_handle.type = gfx::NATIVE_PIXMAP;
gmb_handle.id = gfx::GpuMemoryBufferId(kArbitraryBufferId);
StartCapture(0, params_small_);
SimulateGpuMemoryBufferCreated(kArbitraryBufferId, std::move(gmb_handle));
SimulateBufferReceived(kArbitraryBufferId,
params_small_.requested_format.frame_size,
media::PIXEL_FORMAT_NV12);
};
// The second half of the test: Stop capture and destroy the buffer.
// Everything should happen on |testing_io_thread| here.
auto stop_capture_and_destroy_buffer = [&]() {
StopCapture(0);
SimulateBufferDestroyed(kArbitraryBufferId);
// Explicitly destroy |video_capture_impl_| to make sure it's destroyed on
// the right thread.
video_capture_impl_.reset();
};
testing_io_thread.Start();
testing_io_thread.task_runner()->PostTask(
FROM_HERE, base::BindLambdaForTesting(create_and_queue_buffer));
// Wait until OnFrameReady is called on |testing_io_thread|.
EXPECT_TRUE(frame_ready_event.TimedWait(base::TimeDelta::FromSeconds(3)));
testing_io_thread.task_runner()->PostTask(
FROM_HERE, base::BindLambdaForTesting(stop_capture_and_destroy_buffer));
testing_io_thread.Stop();
EXPECT_EQ(mock_video_capture_host_.released_buffer_count(), 0);
}
TEST_F(VideoCaptureImplTest, BufferReceivedAfterStop) {
const int kArbitraryBufferId = 12;
@ -414,6 +500,33 @@ TEST_F(VideoCaptureImplTest, BufferReceivedAfterStop_ReadOnlyShmemRegion) {
EXPECT_EQ(mock_video_capture_host_.released_buffer_count(), 1);
}
TEST_F(VideoCaptureImplTest, BufferReceivedAfterStop_GpuMemoryBufferHandle) {
const int kArbitraryBufferId = 12;
gfx::GpuMemoryBufferHandle gmb_handle;
gmb_handle.type = gfx::NATIVE_PIXMAP;
gmb_handle.id = gfx::GpuMemoryBufferId(kArbitraryBufferId);
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STOPPED));
EXPECT_CALL(*this, OnFrameReady(_, _)).Times(0);
EXPECT_CALL(mock_video_capture_host_, DoStart(_, session_id_, params_large_));
EXPECT_CALL(mock_video_capture_host_, Stop(_));
EXPECT_CALL(mock_video_capture_host_,
ReleaseBuffer(_, kArbitraryBufferId, _));
StartCapture(0, params_large_);
SimulateGpuMemoryBufferCreated(kArbitraryBufferId, std::move(gmb_handle));
StopCapture(0);
// A buffer received after StopCapture() triggers an instant ReleaseBuffer().
SimulateBufferReceived(kArbitraryBufferId,
params_small_.requested_format.frame_size,
media::PIXEL_FORMAT_NV12);
SimulateBufferDestroyed(kArbitraryBufferId);
EXPECT_EQ(mock_video_capture_host_.released_buffer_count(), 1);
}
TEST_F(VideoCaptureImplTest, AlreadyStarted) {
media::VideoCaptureParams params = {};
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
@ -480,11 +593,15 @@ TEST_F(VideoCaptureImplTest, BufferReceivedBeforeOnStarted) {
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
// Additional STARTED will cause RequestRefreshFrame a second time.
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STOPPED));
EXPECT_CALL(mock_video_capture_host_, Stop(_));
@ -514,11 +631,52 @@ TEST_F(VideoCaptureImplTest,
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
// Additional STARTED will cause RequestRefreshFrame a second time.
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STOPPED));
EXPECT_CALL(mock_video_capture_host_, Stop(_));
StopCapture(0);
}
TEST_F(VideoCaptureImplTest,
BufferReceivedBeforeOnStarted_GpuMemoryBufferHandle) {
const int kArbitraryBufferId = 16;
gfx::GpuMemoryBufferHandle gmb_handle;
gmb_handle.type = gfx::NATIVE_PIXMAP;
gmb_handle.id = gfx::GpuMemoryBufferId(kArbitraryBufferId);
InSequence s;
EXPECT_CALL(mock_video_capture_host_, DoStart(_, session_id_, params_small_))
.WillOnce(DoNothing());
EXPECT_CALL(mock_video_capture_host_,
ReleaseBuffer(_, kArbitraryBufferId, _));
StartCapture(0, params_small_);
SimulateGpuMemoryBufferCreated(kArbitraryBufferId, std::move(gmb_handle));
SimulateBufferReceived(kArbitraryBufferId,
params_small_.requested_format.frame_size,
media::PIXEL_FORMAT_NV12);
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
// Additional STARTED will cause RequestRefreshFrame a second time.
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STARTED));
EXPECT_CALL(mock_video_capture_host_, RequestRefreshFrame(_));
video_capture_impl_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
Mock::VerifyAndClearExpectations(this);
Mock::VerifyAndClearExpectations(&mock_video_capture_host_);
EXPECT_CALL(*this, OnStateUpdate(blink::VIDEO_CAPTURE_STATE_STOPPED));
EXPECT_CALL(mock_video_capture_host_, Stop(_));

@ -21,6 +21,25 @@ GpuMemoryBufferHandle& GpuMemoryBufferHandle::operator=(
GpuMemoryBufferHandle::~GpuMemoryBufferHandle() = default;
GpuMemoryBufferHandle GpuMemoryBufferHandle::Clone() const {
GpuMemoryBufferHandle handle;
handle.type = type;
handle.id = id;
handle.region = region.Duplicate();
handle.offset = offset;
handle.stride = stride;
#if defined(OS_LINUX) || defined(OS_FUCHSIA)
handle.native_pixmap_handle = CloneHandleForIPC(native_pixmap_handle);
#elif defined(OS_MACOSX) && !defined(OS_IOS)
NOTIMPLEMENTED();
#elif defined(OS_WIN)
NOTIMPLEMENTED();
#elif defined(OS_ANDROID)
NOTIMPLEMENTED();
#endif
return handle;
}
void GpuMemoryBuffer::SetColorSpace(const gfx::ColorSpace& color_space) {}
} // namespace gfx

@ -59,6 +59,7 @@ struct GFX_EXPORT GpuMemoryBufferHandle {
GpuMemoryBufferHandle(GpuMemoryBufferHandle&& other);
GpuMemoryBufferHandle& operator=(GpuMemoryBufferHandle&& other);
~GpuMemoryBufferHandle();
GpuMemoryBufferHandle Clone() const;
bool is_null() const { return type == EMPTY_BUFFER; }
GpuMemoryBufferType type;
GpuMemoryBufferId id;

@ -0,0 +1,24 @@
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
mojom = "//ui/gfx/mojom/buffer_types.mojom"
public_headers = [
"//ui/gfx/buffer_types.h",
"//ui/gfx/gpu_memory_buffer.h",
"//ui/gfx/native_pixmap_handle.h",
]
traits_headers = [ "//ui/gfx/mojom/buffer_types_mojom_traits.h" ]
public_deps = [
"//ui/gfx/mojom",
]
type_mappings = [
"gfx.mojom.BufferFormat=gfx::BufferFormat",
"gfx.mojom.BufferUsage=gfx::BufferUsage",
"gfx.mojom.BufferUsageAndFormat=gfx::BufferUsageAndFormat",
"gfx.mojom.GpuMemoryBufferHandle=gfx::GpuMemoryBufferHandle[move_only,nullable_is_same_type]",
"gfx.mojom.GpuMemoryBufferId=gfx::GpuMemoryBufferId[copyable_pass_by_value]",
"gfx.mojom.GpuMemoryBufferType=gfx::GpuMemoryBufferType",
"gfx.mojom.NativePixmapHandle=gfx::NativePixmapHandle[move_only]",
"gfx.mojom.NativePixmapPlane=gfx::NativePixmapPlane[move_only]",
]