media: Add "Search Video Frame with <Search Provider>" support
This CL adds a context menu item for <video> to search for the video frame with the search provider. When Google is the search provider, Google Lens will be used. When the <video> is protected content, this option is disabled. This feature is behind a feature kContextMenuSearchForVideoFrame which is disabled by default. Manually tested on Linux, Windows and Mac in a Chrome build and verified the video frame search with Lens is working well. Also manually tested with a Chromium build and verified the video frame search with the Google or Bing are working well. Browser tests are added for the context menu operations. TODO in followup changes: - Scale the image before sending to search provider. - Add new Lens entry point for this feature. Design doc: go/video-frame-search Bug: 1453681 Test: See above Change-Id: I19238f530378ea4e56ee73d681c63454aa2cab68 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4616044 Reviewed-by: Daniel Cheng <dcheng@chromium.org> Reviewed-by: Evan Liu <evliu@google.com> Reviewed-by: Juan Mojica <juanmojica@google.com> Commit-Queue: Xiaohan Wang <xhwang@chromium.org> Reviewed-by: Dale Curtis <dalecurtis@chromium.org> Reviewed-by: Avi Drissman <avi@chromium.org> Cr-Commit-Position: refs/heads/main@{#1225606}
This commit is contained in:

committed by
Chromium LUCI CQ

parent
f87dbc3be5
commit
b3b99316f6
chrome
content
browser
renderer_host
public
media/base
third_party/blink
public
renderer
tools
blinkpy
presubmit
tools/metrics
ui/gfx/image/mojom
@ -346,8 +346,10 @@
|
||||
#define IDC_CONTENT_CONTEXT_SAVEAVAS 50121
|
||||
#define IDC_CONTENT_CONTEXT_COPYAVLOCATION 50122
|
||||
#define IDC_CONTENT_CONTEXT_COPYVIDEOFRAME 50123
|
||||
#define IDC_CONTENT_CONTEXT_OPENAVNEWTAB 50124
|
||||
#define IDC_CONTENT_CONTEXT_PICTUREINPICTURE 50125
|
||||
#define IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME 50124
|
||||
#define IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME 50125
|
||||
#define IDC_CONTENT_CONTEXT_OPENAVNEWTAB 50126
|
||||
#define IDC_CONTENT_CONTEXT_PICTUREINPICTURE 50127
|
||||
// Media items.
|
||||
#define IDC_CONTENT_CONTEXT_LOOP 50130
|
||||
#define IDC_CONTENT_CONTEXT_CONTROLS 50131
|
||||
|
@ -715,6 +715,9 @@ are declared in tools/grit/grit_rule.gni.
|
||||
<message name="IDS_CONTENT_CONTEXT_COPYVIDEOFRAME" desc="The name of the Copy Video Frame command in the content area context menu">
|
||||
C&opy video frame
|
||||
</message>
|
||||
<message name="IDS_CONTENT_CONTEXT_SEARCHFORVIDEOFRAME" desc="In Title Case: The name of the Lens Search command in the content area context menu">
|
||||
Search video frame with <ph name="VISUAL_SEARCH_PROVIDER">$1<ex>Google Lens</ex></ph>
|
||||
</message>
|
||||
<message name="IDS_CONTENT_CONTEXT_OPENVIDEONEWTAB" desc="The name of the Open Video in New Tab command in the content area context menu">
|
||||
&Open video in new tab
|
||||
</message>
|
||||
@ -987,6 +990,9 @@ are declared in tools/grit/grit_rule.gni.
|
||||
<message name="IDS_CONTENT_CONTEXT_COPYVIDEOFRAME" desc="The name of the Copy Video Frame command in the content area context menu">
|
||||
C&opy Video Frame
|
||||
</message>
|
||||
<message name="IDS_CONTENT_CONTEXT_SEARCHFORVIDEOFRAME" desc="In Title Case: The name of the Lens Search command in the content area context menu">
|
||||
Search Video Frame with <ph name="VISUAL_SEARCH_PROVIDER">$1<ex>Google Lens</ex></ph>
|
||||
</message>
|
||||
<message name="IDS_CONTENT_CONTEXT_OPENVIDEONEWTAB" desc="In Title Case: The name of the Open Video in New Tab command in the content area context menu">
|
||||
&Open Video in New Tab
|
||||
</message>
|
||||
|
@ -0,0 +1 @@
|
||||
6ca20dea9b8423386fb36a0e5dca7621137ea119
|
@ -139,6 +139,7 @@
|
||||
#include "components/guest_view/browser/guest_view_base.h"
|
||||
#include "components/language/core/browser/language_model_manager.h"
|
||||
#include "components/lens/lens_features.h"
|
||||
#include "components/lens/lens_metadata.mojom.h"
|
||||
#include "components/lens/lens_metrics.h"
|
||||
#include "components/live_caption/caption_util.h"
|
||||
#include "components/live_caption/pref_names.h"
|
||||
@ -511,13 +512,15 @@ const std::map<int, int>& GetIdcToUmaMap(UmaEnumIdLookupType type) {
|
||||
{IDC_CONTEXT_COMPOSE, 139},
|
||||
{IDC_CONTENT_CONTEXT_AUTOFILL_FALLBACK_PAYMENTS, 140},
|
||||
{IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS, 141},
|
||||
{IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME, 142},
|
||||
{IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME, 143},
|
||||
// To add new items:
|
||||
// - Add one more line above this comment block, using the UMA value
|
||||
// from the line below this comment block.
|
||||
// - Increment the UMA value in that latter line.
|
||||
// - Add the new item to the RenderViewContextMenuItem enum in
|
||||
// tools/metrics/histograms/enums.xml.
|
||||
{0, 142}});
|
||||
{0, 144}});
|
||||
|
||||
// These UMA values are for the the ContextMenuOptionDesktop enum, used for
|
||||
// the ContextMenu.SelectedOptionDesktop histograms.
|
||||
@ -1323,6 +1326,12 @@ int RenderViewContextMenu::GetRegionSearchIdc() const {
|
||||
: IDC_CONTENT_CONTEXT_WEB_REGION_SEARCH;
|
||||
}
|
||||
|
||||
int RenderViewContextMenu::GetSearchForVideoFrameIdc() const {
|
||||
return search::DefaultSearchProviderIsGoogle(GetProfile())
|
||||
? IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME
|
||||
: IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME;
|
||||
}
|
||||
|
||||
const TemplateURL* RenderViewContextMenu::GetImageSearchProvider() const {
|
||||
#if BUILDFLAG(IS_CHROMEOS_ASH)
|
||||
if (!crosapi::browser_util::IsAshWebBrowserEnabled()) {
|
||||
@ -1996,6 +2005,23 @@ void RenderViewContextMenu::AppendVideoItems() {
|
||||
menu_model_.AddItemWithStringId(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME,
|
||||
IDS_CONTENT_CONTEXT_COPYVIDEOFRAME);
|
||||
}
|
||||
|
||||
if (base::FeatureList::IsEnabled(media::kContextMenuSearchForVideoFrame)) {
|
||||
const auto* provider = GetImageSearchProvider();
|
||||
if (!provider) {
|
||||
return;
|
||||
}
|
||||
|
||||
menu_model_.AddItem(
|
||||
GetSearchForVideoFrameIdc(),
|
||||
l10n_util::GetStringFUTF16(IDS_CONTENT_CONTEXT_SEARCHFORVIDEOFRAME,
|
||||
GetImageSearchProviderName(provider)));
|
||||
if (companion::IsNewBadgeEnabledForSearchMenuItem(GetBrowser())) {
|
||||
menu_model_.SetIsNewFeatureAt(menu_model_.GetItemCount() - 1, true);
|
||||
}
|
||||
MaybePrepareForLensQuery();
|
||||
}
|
||||
|
||||
menu_model_.AddItemWithStringId(IDC_CONTENT_CONTEXT_COPYAVLOCATION,
|
||||
IDS_CONTENT_CONTEXT_COPYVIDEOLOCATION);
|
||||
menu_model_.AddCheckItemWithStringId(IDC_CONTENT_CONTEXT_PICTUREINPICTURE,
|
||||
@ -2709,6 +2735,8 @@ bool RenderViewContextMenu::IsCommandIdEnabled(int id) const {
|
||||
|
||||
case IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS:
|
||||
case IDC_CONTENT_CONTEXT_COPYVIDEOFRAME:
|
||||
case IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME:
|
||||
case IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME:
|
||||
return (params_.media_flags & ContextMenuData::kMediaEncrypted) == 0 &&
|
||||
(params_.media_flags &
|
||||
ContextMenuData::kMediaHasReadableVideoFrame) != 0;
|
||||
@ -3048,6 +3076,11 @@ void RenderViewContextMenu::ExecuteCommand(int id, int event_flags) {
|
||||
ExecCopyVideoFrame();
|
||||
break;
|
||||
|
||||
case IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME:
|
||||
case IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME:
|
||||
ExecSearchForVideoFrame();
|
||||
break;
|
||||
|
||||
case IDC_CONTENT_CONTEXT_SEARCHWEBFORIMAGE:
|
||||
ExecSearchWebForImage(/*is_image_translate=*/false);
|
||||
break;
|
||||
@ -4157,16 +4190,29 @@ void RenderViewContextMenu::ExecSaveVideoFrameAs() {
|
||||
gfx::Point(params_.x, params_.y),
|
||||
blink::mojom::MediaPlayerAction(
|
||||
blink::mojom::MediaPlayerActionType::kSaveVideoFrameAs,
|
||||
!IsCommandIdChecked(IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS)));
|
||||
/*enable=*/true));
|
||||
}
|
||||
|
||||
void RenderViewContextMenu::ExecCopyVideoFrame() {
|
||||
base::RecordAction(UserMetricsAction("MediaContextMenu_CopyVideoFrame"));
|
||||
MediaPlayerActionAt(
|
||||
MediaPlayerActionAt(gfx::Point(params_.x, params_.y),
|
||||
blink::mojom::MediaPlayerAction(
|
||||
blink::mojom::MediaPlayerActionType::kCopyVideoFrame,
|
||||
/*enable=*/true));
|
||||
}
|
||||
|
||||
void RenderViewContextMenu::ExecSearchForVideoFrame() {
|
||||
base::RecordAction(UserMetricsAction("MediaContextMenu_SearchForVideoFrame"));
|
||||
|
||||
RenderFrameHost* frame_host = GetRenderFrameHost();
|
||||
if (!frame_host) {
|
||||
return;
|
||||
}
|
||||
|
||||
frame_host->RequestVideoFrameAt(
|
||||
gfx::Point(params_.x, params_.y),
|
||||
blink::mojom::MediaPlayerAction(
|
||||
blink::mojom::MediaPlayerActionType::kCopyVideoFrame,
|
||||
!IsCommandIdChecked(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME)));
|
||||
base::BindOnce(&RenderViewContextMenu::SearchForVideoFrame,
|
||||
weak_pointer_factory_.GetWeakPtr()));
|
||||
}
|
||||
|
||||
void RenderViewContextMenu::ExecLiveCaption() {
|
||||
@ -4320,6 +4366,27 @@ void RenderViewContextMenu::MediaPlayerActionAt(
|
||||
frame_host->ExecuteMediaPlayerActionAtLocation(location, action);
|
||||
}
|
||||
|
||||
void RenderViewContextMenu::SearchForVideoFrame(const gfx::ImageSkia& image) {
|
||||
if (image.isNull()) {
|
||||
return;
|
||||
}
|
||||
|
||||
CoreTabHelper* core_tab_helper =
|
||||
CoreTabHelper::FromWebContents(source_web_contents_);
|
||||
if (!core_tab_helper) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO(crbug.com/1453681): Add an entry point for VideoFrame search.
|
||||
if (search::DefaultSearchProviderIsGoogle(GetProfile())) {
|
||||
core_tab_helper->SearchWithLens(
|
||||
gfx::Image(image), image.size(), {},
|
||||
lens::EntryPoint::CHROME_REGION_SEARCH_MENU_ITEM);
|
||||
} else {
|
||||
core_tab_helper->SearchByImage(gfx::Image(image), image.size());
|
||||
}
|
||||
}
|
||||
|
||||
void RenderViewContextMenu::PluginActionAt(
|
||||
const gfx::Point& location,
|
||||
blink::mojom::PluginActionType plugin_action) {
|
||||
|
@ -173,6 +173,9 @@ class RenderViewContextMenu
|
||||
// Returns the correct IDC for the Region Search context menu string
|
||||
int GetRegionSearchIdc() const;
|
||||
|
||||
// Returns the correct IDC for the Video Frame Search context menu string
|
||||
int GetSearchForVideoFrameIdc() const;
|
||||
|
||||
// Returns the provider for image search.
|
||||
const TemplateURL* GetImageSearchProvider() const;
|
||||
|
||||
@ -377,6 +380,7 @@ class RenderViewContextMenu
|
||||
void ExecControls();
|
||||
void ExecSaveVideoFrameAs();
|
||||
void ExecCopyVideoFrame();
|
||||
void ExecSearchForVideoFrame();
|
||||
void ExecLiveCaption();
|
||||
void ExecRotateCW();
|
||||
void ExecRotateCCW();
|
||||
@ -396,6 +400,7 @@ class RenderViewContextMenu
|
||||
|
||||
void MediaPlayerActionAt(const gfx::Point& location,
|
||||
const blink::mojom::MediaPlayerAction& action);
|
||||
void SearchForVideoFrame(const gfx::ImageSkia& image);
|
||||
void PluginActionAt(const gfx::Point& location,
|
||||
blink::mojom::PluginActionType plugin_action);
|
||||
|
||||
|
@ -206,7 +206,9 @@ class ContextMenuBrowserTest : public MixinBasedInProcessBrowserTest {
|
||||
protected:
|
||||
ContextMenuBrowserTest() {
|
||||
scoped_feature_list_.InitWithFeatures(
|
||||
{features::kReadAnything, media::kContextMenuSaveVideoFrameAs}, {});
|
||||
{features::kReadAnything, media::kContextMenuSaveVideoFrameAs,
|
||||
media::kContextMenuSearchForVideoFrame},
|
||||
{});
|
||||
}
|
||||
|
||||
std::unique_ptr<TestRenderViewContextMenu> CreateContextMenuMediaTypeNone(
|
||||
@ -2971,8 +2973,11 @@ IN_PROC_BROWSER_TEST_F(ContextMenuBrowserTest,
|
||||
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS));
|
||||
EXPECT_TRUE(menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS));
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS));
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME));
|
||||
EXPECT_TRUE(menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME));
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME));
|
||||
EXPECT_TRUE(
|
||||
menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME));
|
||||
}
|
||||
|
||||
IN_PROC_BROWSER_TEST_F(ContextMenuBrowserTest,
|
||||
@ -2986,6 +2991,9 @@ IN_PROC_BROWSER_TEST_F(ContextMenuBrowserTest,
|
||||
EXPECT_FALSE(menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS));
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME));
|
||||
EXPECT_FALSE(menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_COPYVIDEOFRAME));
|
||||
EXPECT_TRUE(menu->IsItemPresent(IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME));
|
||||
EXPECT_FALSE(
|
||||
menu->IsCommandIdEnabled(IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME));
|
||||
}
|
||||
|
||||
IN_PROC_BROWSER_TEST_F(ContextMenuBrowserTest, ContextMenuForEncryptedVideo) {
|
||||
|
@ -2516,6 +2516,15 @@ void RenderFrameHostImpl::ExecuteMediaPlayerActionAtLocation(
|
||||
std::move(media_player_action));
|
||||
}
|
||||
|
||||
void RenderFrameHostImpl::RequestVideoFrameAt(
|
||||
const gfx::Point& location,
|
||||
base::OnceCallback<void(const gfx::ImageSkia&)> callback) {
|
||||
gfx::PointF point_in_view = GetView()->TransformRootPointToViewCoordSpace(
|
||||
gfx::PointF(location.x(), location.y()));
|
||||
GetAssociatedLocalFrame()->RequestVideoFrameAt(
|
||||
gfx::Point(point_in_view.x(), point_in_view.y()), std::move(callback));
|
||||
}
|
||||
|
||||
bool RenderFrameHostImpl::CreateNetworkServiceDefaultFactory(
|
||||
mojo::PendingReceiver<network::mojom::URLLoaderFactory>
|
||||
default_factory_receiver) {
|
||||
|
@ -497,6 +497,9 @@ class CONTENT_EXPORT RenderFrameHostImpl
|
||||
void ExecuteMediaPlayerActionAtLocation(
|
||||
const gfx::Point&,
|
||||
const blink::mojom::MediaPlayerAction& action) override;
|
||||
void RequestVideoFrameAt(
|
||||
const gfx::Point& viewport_position,
|
||||
base::OnceCallback<void(const gfx::ImageSkia&)> callback) override;
|
||||
bool CreateNetworkServiceDefaultFactory(
|
||||
mojo::PendingReceiver<network::mojom::URLLoaderFactory>
|
||||
default_factory_receiver) override;
|
||||
|
@ -39,6 +39,7 @@
|
||||
#include "third_party/blink/public/mojom/permissions_policy/permissions_policy_feature.mojom-forward.h"
|
||||
#include "third_party/perfetto/include/perfetto/tracing/traced_value_forward.h"
|
||||
#include "ui/accessibility/ax_node_id_forward.h"
|
||||
#include "ui/gfx/image/image_skia.h"
|
||||
#include "ui/gfx/native_widget_types.h"
|
||||
|
||||
class GURL;
|
||||
@ -860,6 +861,10 @@ class CONTENT_EXPORT RenderFrameHost : public IPC::Listener,
|
||||
const gfx::Point& location,
|
||||
const blink::mojom::MediaPlayerAction& action) = 0;
|
||||
|
||||
virtual void RequestVideoFrameAt(
|
||||
const gfx::Point& location,
|
||||
base::OnceCallback<void(const gfx::ImageSkia&)> callback) = 0;
|
||||
|
||||
// Creates a Network Service-backed factory from appropriate |NetworkContext|.
|
||||
//
|
||||
// If this returns true, any redirect safety checks should be bypassed in
|
||||
|
@ -88,6 +88,10 @@ void FakeLocalFrame::MediaPlayerActionAt(
|
||||
const gfx::Point& location,
|
||||
blink::mojom::MediaPlayerActionPtr action) {}
|
||||
|
||||
void FakeLocalFrame::RequestVideoFrameAt(const gfx::Point& window_point,
|
||||
RequestVideoFrameAtCallback callback) {
|
||||
}
|
||||
|
||||
void FakeLocalFrame::PluginActionAt(const gfx::Point& location,
|
||||
blink::mojom::PluginActionType action) {}
|
||||
|
||||
|
@ -70,6 +70,8 @@ class FakeLocalFrame : public blink::mojom::LocalFrame {
|
||||
void BeforeUnload(bool is_reload, BeforeUnloadCallback callback) override;
|
||||
void MediaPlayerActionAt(const gfx::Point& location,
|
||||
blink::mojom::MediaPlayerActionPtr action) override;
|
||||
void RequestVideoFrameAt(const gfx::Point& window_point,
|
||||
RequestVideoFrameAtCallback callback) override;
|
||||
void PluginActionAt(const gfx::Point& location,
|
||||
blink::mojom::PluginActionType action) override;
|
||||
void AdvanceFocusInFrame(blink::mojom::FocusType focus_type,
|
||||
|
@ -448,6 +448,11 @@ BASE_FEATURE(kContextMenuSaveVideoFrameAs,
|
||||
"ContextMenuSaveVideoFrameAs",
|
||||
base::FEATURE_DISABLED_BY_DEFAULT);
|
||||
|
||||
// Enables the "Search Video Frame with <Search Provider>" context menu item.
|
||||
BASE_FEATURE(kContextMenuSearchForVideoFrame,
|
||||
"ContextMenuSearchForVideoFrame",
|
||||
base::FEATURE_DISABLED_BY_DEFAULT);
|
||||
|
||||
// Enables the "Copy Video Frame" context menu item.
|
||||
BASE_FEATURE(kContextMenuCopyVideoFrame,
|
||||
"ContextMenuCopyVideoFrame",
|
||||
|
@ -187,6 +187,7 @@ MEDIA_EXPORT BASE_DECLARE_FEATURE(kCdmHostVerification);
|
||||
MEDIA_EXPORT BASE_DECLARE_FEATURE(kCdmProcessSiteIsolation);
|
||||
MEDIA_EXPORT BASE_DECLARE_FEATURE(kContextMenuSaveVideoFrameAs);
|
||||
MEDIA_EXPORT BASE_DECLARE_FEATURE(kContextMenuCopyVideoFrame);
|
||||
MEDIA_EXPORT BASE_DECLARE_FEATURE(kContextMenuSearchForVideoFrame);
|
||||
#if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
|
||||
MEDIA_EXPORT BASE_DECLARE_FEATURE(kChromeWideEchoCancellation);
|
||||
MEDIA_EXPORT extern const base::FeatureParam<bool>
|
||||
|
2
third_party/blink/public/mojom/BUILD.gn
vendored
2
third_party/blink/public/mojom/BUILD.gn
vendored
@ -318,6 +318,7 @@ mojom("mojom_platform") {
|
||||
"//ui/events/mojom",
|
||||
"//ui/events/mojom:event_latency_metadata_mojom",
|
||||
"//ui/gfx/geometry/mojom",
|
||||
"//ui/gfx/image/mojom",
|
||||
"//ui/gfx/mojom",
|
||||
"//ui/gfx/range/mojom",
|
||||
"//url/mojom:url_mojom_gurl",
|
||||
@ -1347,6 +1348,7 @@ mojom("mojom_core") {
|
||||
"//ui/color:mojom",
|
||||
"//ui/display/mojom",
|
||||
"//ui/gfx/geometry/mojom",
|
||||
"//ui/gfx/image/mojom",
|
||||
"//ui/gfx/mojom",
|
||||
"//ui/gfx/range/mojom",
|
||||
"//url/mojom:url_mojom_gurl",
|
||||
|
@ -72,6 +72,7 @@ import "ui/base/ime/mojom/virtual_keyboard_types.mojom";
|
||||
import "ui/base/mojom/window_open_disposition.mojom";
|
||||
import "ui/events/mojom/scroll_granularity.mojom";
|
||||
import "ui/gfx/geometry/mojom/geometry.mojom";
|
||||
import "ui/gfx/image/mojom/image.mojom";
|
||||
import "ui/gfx/range/mojom/range.mojom";
|
||||
import "url/mojom/origin.mojom";
|
||||
import "url/mojom/url.mojom";
|
||||
@ -913,6 +914,10 @@ interface LocalFrame {
|
||||
// at the given point in the view coordinate space.
|
||||
MediaPlayerActionAt(gfx.mojom.Point location, blink.mojom.MediaPlayerAction action);
|
||||
|
||||
// Requests the current video frame (as `image`) from the media player at the
|
||||
// given `location`.
|
||||
RequestVideoFrameAt(gfx.mojom.Point location) => (gfx.mojom.ImageSkia image);
|
||||
|
||||
// Tells the renderer to perform the given action on the plugin location at
|
||||
// the given point in the view coordinate space.
|
||||
PluginActionAt(gfx.mojom.Point location, blink.mojom.PluginActionType action);
|
||||
|
@ -4,7 +4,7 @@
|
||||
|
||||
module blink.mojom;
|
||||
|
||||
// Perform the below mention actions on the media player
|
||||
// Perform the below actions on the media player.
|
||||
enum MediaPlayerActionType {
|
||||
kLoop,
|
||||
kControls,
|
||||
@ -14,10 +14,11 @@ enum MediaPlayerActionType {
|
||||
kDefaultActionType = kPictureInPicture
|
||||
};
|
||||
|
||||
// Send MediaPlayerAction data from browser to renderer. Attribute
|
||||
// 'type' defines action needs to be performed on Media Player and
|
||||
// boolean attribute 'enable' enables or disables the action.
|
||||
// Send MediaPlayerAction data from browser to renderer. Attribute `type`
|
||||
// defines action needs to be performed on Media Player and boolean attribute
|
||||
// `enable` indicates enabling or disabling the option, e.g. for `kLoop`,
|
||||
// `kControls` and `kPictureInPicture`. For other actions, `enable` is ignored.
|
||||
struct MediaPlayerAction {
|
||||
MediaPlayerActionType type = kDefaultActionType;
|
||||
bool enable = false;
|
||||
};
|
||||
};
|
||||
|
1
third_party/blink/renderer/core/frame/DEPS
vendored
1
third_party/blink/renderer/core/frame/DEPS
vendored
@ -34,6 +34,7 @@ specific_include_rules = {
|
||||
"+base/run_loop.h",
|
||||
],
|
||||
"local_frame\.h": [
|
||||
"+ui/gfx/image/image_skia.h",
|
||||
"+ui/gfx/transform.h"
|
||||
],
|
||||
"local_frame\.cc": [
|
||||
|
@ -3401,6 +3401,43 @@ void LocalFrame::MediaPlayerActionAtViewportPoint(
|
||||
}
|
||||
}
|
||||
|
||||
void LocalFrame::RequestVideoFrameAt(
|
||||
const gfx::Point& viewport_position,
|
||||
base::OnceCallback<void(const gfx::ImageSkia&)> callback) {
|
||||
HitTestResult result = HitTestResultForVisualViewportPos(viewport_position);
|
||||
Node* node = result.InnerNode();
|
||||
auto* video = DynamicTo<HTMLVideoElement>(node);
|
||||
|
||||
if (!video) {
|
||||
std::move(callback).Run(gfx::ImageSkia());
|
||||
return;
|
||||
}
|
||||
|
||||
auto image = video->CreateStaticBitmapImage();
|
||||
if (!image) {
|
||||
std::move(callback).Run(gfx::ImageSkia());
|
||||
return;
|
||||
}
|
||||
|
||||
auto bitmap = image->AsSkBitmapForCurrentFrame(kRespectImageOrientation);
|
||||
|
||||
// Only kN32_SkColorType bitmaps can be sent across IPC, so convert if
|
||||
// necessary.
|
||||
SkBitmap converted_bitmap;
|
||||
if (bitmap.colorType() == kN32_SkColorType) {
|
||||
converted_bitmap = bitmap;
|
||||
} else {
|
||||
SkImageInfo info = bitmap.info().makeColorType(kN32_SkColorType);
|
||||
if (converted_bitmap.tryAllocPixels(info)) {
|
||||
bitmap.readPixels(info, converted_bitmap.getPixels(),
|
||||
converted_bitmap.rowBytes(), 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
std::move(callback).Run(gfx::ImageSkia::CreateFromBitmap(converted_bitmap,
|
||||
/*scale=*/1));
|
||||
}
|
||||
|
||||
void LocalFrame::DownloadURL(
|
||||
const ResourceRequest& request,
|
||||
network::mojom::blink::RedirectMode cross_origin_redirect_behavior) {
|
||||
|
@ -96,6 +96,7 @@
|
||||
#include "third_party/blink/renderer/platform/supplementable.h"
|
||||
#include "third_party/perfetto/include/perfetto/tracing/traced_value_forward.h"
|
||||
#include "ui/gfx/geometry/transform.h"
|
||||
#include "ui/gfx/image/image_skia.h"
|
||||
|
||||
namespace base {
|
||||
class SingleThreadTaskRunner;
|
||||
@ -731,6 +732,9 @@ class CORE_EXPORT LocalFrame final
|
||||
const gfx::Point& viewport_position,
|
||||
const blink::mojom::blink::MediaPlayerActionType type,
|
||||
bool enable);
|
||||
void RequestVideoFrameAt(
|
||||
const gfx::Point& viewport_position,
|
||||
base::OnceCallback<void(const gfx::ImageSkia&)> callback);
|
||||
|
||||
// Handle the request as a download. If the request is for a blob: URL,
|
||||
// a BlobURLToken should be provided as |blob_url_token| to ensure the
|
||||
|
@ -696,6 +696,14 @@ void LocalFrameMojoHandler::MediaPlayerActionAt(
|
||||
action->enable);
|
||||
}
|
||||
|
||||
void LocalFrameMojoHandler::RequestVideoFrameAt(
|
||||
const gfx::Point& window_point,
|
||||
RequestVideoFrameAtCallback callback) {
|
||||
gfx::Point viewport_position =
|
||||
frame_->GetWidgetForLocalRoot()->DIPsToRoundedBlinkSpace(window_point);
|
||||
frame_->RequestVideoFrameAt(viewport_position, std::move(callback));
|
||||
}
|
||||
|
||||
void LocalFrameMojoHandler::AdvanceFocusInFrame(
|
||||
mojom::blink::FocusType focus_type,
|
||||
const absl::optional<RemoteFrameToken>& source_frame_token) {
|
||||
|
@ -120,9 +120,10 @@ class LocalFrameMojoHandler
|
||||
void ReportBlinkFeatureUsage(const Vector<mojom::blink::WebFeature>&) final;
|
||||
void RenderFallbackContent() final;
|
||||
void BeforeUnload(bool is_reload, BeforeUnloadCallback callback) final;
|
||||
void MediaPlayerActionAt(
|
||||
const gfx::Point& window_point,
|
||||
blink::mojom::blink::MediaPlayerActionPtr action) final;
|
||||
void MediaPlayerActionAt(const gfx::Point& window_point,
|
||||
mojom::blink::MediaPlayerActionPtr action) final;
|
||||
void RequestVideoFrameAt(const gfx::Point& window_point,
|
||||
RequestVideoFrameAtCallback callback) final;
|
||||
void AdvanceFocusInFrame(
|
||||
mojom::blink::FocusType focus_type,
|
||||
const absl::optional<RemoteFrameToken>& source_frame_token) final;
|
||||
|
@ -1754,6 +1754,15 @@ _CONFIG = [
|
||||
'storage::GetIdentifierFromOrigin',
|
||||
],
|
||||
},
|
||||
{
|
||||
'paths': [
|
||||
'third_party/blink/renderer/core/frame/local_frame.cc',
|
||||
'third_party/blink/renderer/core/frame/local_frame.h',
|
||||
],
|
||||
'allowed': [
|
||||
'gfx::ImageSkia',
|
||||
],
|
||||
},
|
||||
{
|
||||
'paths': ['third_party/blink/renderer/core/frame/local_frame_view.cc'],
|
||||
'allowed': [
|
||||
|
@ -18802,6 +18802,15 @@ should be able to be added at any place in this file.
|
||||
</description>
|
||||
</action>
|
||||
|
||||
<action name="MediaContextMenu_SearchForVideoFrame">
|
||||
<owner>xhwang@chromium.org</owner>
|
||||
<owner>media-dev@chromium.org</owner>
|
||||
<description>
|
||||
User clicked on the contextual menu of a video player to search for the
|
||||
video frame.
|
||||
</description>
|
||||
</action>
|
||||
|
||||
<action name="MediaIndicatorButton_Dragged">
|
||||
<obsolete>Replaced with "AlertIndicatorButton_Dragged".</obsolete>
|
||||
<owner>miu@chromium.org</owner>
|
||||
|
@ -54692,6 +54692,8 @@ https://www.dmtf.org/sites/default/files/standards/documents/DSP0134_2.7.1.pdf
|
||||
<int value="139" label="IDC_CONTEXT_COMPOSE"/>
|
||||
<int value="140" label="IDC_CONTEXT_CONTEXT_AUTOFILL_FALLBACK_PAYMENTS"/>
|
||||
<int value="141" label="IDC_CONTENT_CONTEXT_SAVEVIDEOFRAMEAS"/>
|
||||
<int value="142" label="IDC_CONTENT_CONTEXT_SEARCHLENSFORVIDEOFRAME"/>
|
||||
<int value="143" label="IDC_CONTENT_CONTEXT_SEARCHWEBFORVIDEOFRAME"/>
|
||||
</enum>
|
||||
|
||||
<enum name="ReportCheckMembershipResponseCases">
|
||||
|
@ -5,6 +5,7 @@
|
||||
import("//mojo/public/tools/bindings/mojom.gni")
|
||||
|
||||
mojom("mojom") {
|
||||
generate_java = true
|
||||
sources = [ "image.mojom" ]
|
||||
|
||||
webui_module_path = "chrome://resources/mojo/ui/gfx/image/mojom"
|
||||
@ -14,23 +15,24 @@ mojom("mojom") {
|
||||
|
||||
public_deps = [ "//skia/public/mojom" ]
|
||||
|
||||
cpp_typemaps = [
|
||||
{
|
||||
types = [
|
||||
{
|
||||
mojom = "gfx.mojom.ImageSkia"
|
||||
cpp = "::gfx::ImageSkia"
|
||||
nullable_is_same_type = true
|
||||
},
|
||||
{
|
||||
mojom = "gfx.mojom.ImageSkiaRep"
|
||||
cpp = "::gfx::ImageSkiaRep"
|
||||
},
|
||||
]
|
||||
traits_headers = [ "image_skia_mojom_traits.h" ]
|
||||
traits_public_deps = [ ":mojom_traits" ]
|
||||
},
|
||||
]
|
||||
shared_cpp_typemap = {
|
||||
types = [
|
||||
{
|
||||
mojom = "gfx.mojom.ImageSkia"
|
||||
cpp = "::gfx::ImageSkia"
|
||||
nullable_is_same_type = true
|
||||
},
|
||||
{
|
||||
mojom = "gfx.mojom.ImageSkiaRep"
|
||||
cpp = "::gfx::ImageSkiaRep"
|
||||
},
|
||||
]
|
||||
traits_headers = [ "image_skia_mojom_traits.h" ]
|
||||
traits_public_deps = [ ":mojom_traits" ]
|
||||
}
|
||||
|
||||
cpp_typemaps = [ shared_cpp_typemap ]
|
||||
blink_cpp_typemaps = [ shared_cpp_typemap ]
|
||||
}
|
||||
|
||||
source_set("mojom_traits") {
|
||||
|
Reference in New Issue
Block a user