[lensoverlay] Send PDF bytes for contextualization

When the feature flag is enabled, extracts the PDF bytes from the PDF
renderer process and sends them to the Lens server to help with
contextualization. This CL is meant to help prototype and unblock server
side work, not be used on any real users yet. Therefore the logic is
gated by a feature flag defaulted to off. Bytes only leave the device if
the user invokes the Lens Overlay after manually turning on the feature
flag.

Before enabling the flag for real users, updates will be made to our
privacy data notice so no bytes leave the device unless the user
consents.

More info in the design doc linked in the bug.

Bug: 361409401
AX-Relnotes: n/a.
Change-Id: I9c261fe4e02a607a845bcef6bb2375830880ff37
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5811891
Reviewed-by: Lei Zhang <thestig@chromium.org>
Commit-Queue: Duncan Mercer <mercerd@google.com>
Reviewed-by: Jason Hu <hujasonx@google.com>
Cr-Commit-Position: refs/heads/main@{#1350343}

chrome/browser/ui/lens/BUILD.gn

@@ -111,6 +111,7 @@ source_set("lens") {
     "//components/lens:features",
     "//components/lens/proto/server:proto",
     "//components/metrics_services_manager:metrics_services_manager",
+    "//components/pdf/browser",
     "//components/prefs",
     "//components/sessions",
     "//components/signin/public/identity_manager",

chrome/browser/ui/lens/lens_overlay_controller.cc

@@ -45,6 +45,7 @@
 #include "components/find_in_page/find_tab_helper.h"
 #include "components/lens/lens_features.h"
 #include "components/lens/lens_overlay_permission_utils.h"
+#include "components/pdf/browser/pdf_document_helper.h"
 #include "components/permissions/permission_request_manager.h"
 #include "components/sessions/content/session_tab_helper.h"
 #include "components/signin/public/identity_manager/identity_manager.h"
@@ -181,6 +182,28 @@ SkBitmap CreateRgbBitmap(const SkBitmap& bgr_bitmap) {
   return SkBitmap();
 }
 
+// Returns the PDFHelper associated with the given web contents. Returns nullptr
+// if one does not exist.
+pdf::PDFDocumentHelper* MaybeGetPdfHelper(content::WebContents* contents) {
+  pdf::PDFDocumentHelper* pdf_helper = nullptr;
+  // Iterate through each of the render frame hosts, because the frame
+  // associated to a PDFDocumentHelper is not guaranteed to be a specific frame.
+  // For example, if kPdfOopif feature is enabled, the frame is the top frame.
+  // If kPdfOopif is disabled, it is a child frame.
+  contents->ForEachRenderFrameHost(
+      [&pdf_helper](content::RenderFrameHost* rfh) {
+        if (pdf_helper) {
+          return;
+        }
+        auto* possible_pdf_helper =
+            pdf::PDFDocumentHelper::GetForCurrentDocument(rfh);
+        if (possible_pdf_helper) {
+          pdf_helper = possible_pdf_helper;
+        }
+      });
+  return pdf_helper;
+}
+
 }  // namespace
 
 LensOverlayController::LensOverlayController(
@@ -1223,9 +1246,29 @@ void LensOverlayController::ContinueCreateInitializationData(
   initialization_data_ = std::make_unique<OverlayInitializationData>(
       screenshot, std::move(rgb_screenshot), color_palette, page_url,
       page_title);
+
   AddBoundingBoxesToInitializationData(all_bounds);
 
-  // Initialize the overlay now that the initialization data is ready.
+  // Try and fetch the PDF bytes if enabled.
+  pdf::PDFDocumentHelper* pdf_helper =
+      lens::features::UsePdfsAsContext()
+          ? MaybeGetPdfHelper(active_web_contents)
+          : nullptr;
+  if (pdf_helper) {
+    // Fetch the PDF bytes then initialize the overlay.
+    pdf_helper->GetPdfBytes(
+        base::BindOnce(&LensOverlayController::OnPdfBytesReceived,
+                       weak_factory_.GetWeakPtr()));
+    return;
+  }
+
+  // Initialize since there are no PDF bytes to wait on.
+  InitializeOverlay();
+}
+
+void LensOverlayController::OnPdfBytesReceived(
+    const std::vector<uint8_t>& bytes) {
+  initialization_data_->pdf_bytes_ = bytes;
   InitializeOverlay();
 }
 
@@ -1470,6 +1513,7 @@ void LensOverlayController::InitializeOverlay() {
         initialization_data_->current_screenshot_,
         initialization_data_->page_url_, initialization_data_->page_title_,
         std::move(initialization_data_->significant_region_boxes_),
+        initialization_data_->pdf_bytes_,
         device_scale_factor * page_scale_factor);
   }
   // TODO(b/352622136): We should not start the lens request until the overlay

chrome/browser/ui/lens/lens_overlay_controller.h

@@ -553,6 +553,10 @@ class LensOverlayController : public LensSearchboxClient,
     // The page title, if it is allowed to be shared.
     std::optional<std::string> page_title_;
 
+    // The bytes of the PDF the user is viewing, if the user is looking at a PDF
+    // and the bytes are able to be retrieved.
+    std::vector<uint8_t> pdf_bytes_;
+
     // Bounding boxes for significant regions identified in the screenshot.
     std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes_;
 
@@ -628,6 +632,10 @@ class LensOverlayController : public LensSearchboxClient,
       const std::vector<gfx::Rect>& all_bounds,
       SkBitmap rgb_screenshot);
 
+  // Receives the PDF bytes from the IPC call to the PDF renderer and stores
+  // them in initialization data.
+  void OnPdfBytesReceived(const std::vector<uint8_t>& bytes);
+
   // Adds bounding boxes to the initialization data.
   void AddBoundingBoxesToInitializationData(
       const std::vector<gfx::Rect>& bounds);

chrome/browser/ui/lens/lens_overlay_controller_browsertest.cc

@@ -332,6 +332,7 @@ class LensOverlayQueryControllerFake : public lens::LensOverlayQueryController {
       std::optional<GURL> page_url,
       std::optional<std::string> page_title,
       std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes,
+      base::span<const uint8_t> pdf_bytes,
       float ui_scale_factor) override {
     // Send response for full image callback / HandleStartQueryResponse.
     std::vector<lens::mojom::OverlayObjectPtr> test_objects;
@@ -349,6 +350,8 @@ class LensOverlayQueryControllerFake : public lens::LensOverlayQueryController {
     base::SequencedTaskRunner::GetCurrentDefault()->PostTask(
         FROM_HERE,
         base::BindOnce(interaction_data_callback_, interaction_response));
+
+    last_sent_pdf_bytes_ = pdf_bytes;
   }
 
   void SendTaskCompletionGen204IfEnabled(
@@ -406,6 +409,7 @@ class LensOverlayQueryControllerFake : public lens::LensOverlayQueryController {
     last_queried_region_.reset();
     last_queried_text_.clear();
     last_queried_region_bytes_ = std::nullopt;
+    last_sent_pdf_bytes_ = base::span<const uint8_t>();
   }
 
   bool full_image_request_should_return_error_ = false;
@@ -413,6 +417,7 @@ class LensOverlayQueryControllerFake : public lens::LensOverlayQueryController {
   lens::LensOverlaySelectionType last_lens_selection_type_;
   lens::mojom::CenterRotatedBoxPtr last_queried_region_;
   std::optional<SkBitmap> last_queried_region_bytes_;
+  base::span<const uint8_t> last_sent_pdf_bytes_;
   std::optional<lens::mojom::UserAction> last_user_action_;
 };
 
@@ -524,12 +529,15 @@ class TabFeaturesFake : public tabs::TabFeatures {
   }
 };
 
+std::unique_ptr<tabs::TabFeatures> CreateTabFeatures() {
+  return std::make_unique<TabFeaturesFake>();
+}
+
 class LensOverlayControllerBrowserTest : public InProcessBrowserTest {
  protected:
   LensOverlayControllerBrowserTest() {
-    tabs::TabFeatures::ReplaceTabFeaturesForTesting(base::BindRepeating(
-        &LensOverlayControllerBrowserTest::CreateTabFeatures,
-        base::Unretained(this)));
+    tabs::TabFeatures::ReplaceTabFeaturesForTesting(
+        base::BindRepeating(&CreateTabFeatures));
   }
 
   void SetUp() override {
@@ -572,10 +580,6 @@ class LensOverlayControllerBrowserTest : public InProcessBrowserTest {
         });
   }
 
-  std::unique_ptr<tabs::TabFeatures> CreateTabFeatures() {
-    return std::make_unique<TabFeaturesFake>();
-  }
-
   const SkBitmap CreateNonEmptyBitmap(int width, int height) {
     SkBitmap bitmap;
     bitmap.allocN32Pixels(width, height);
@@ -3617,7 +3621,10 @@ class LensOverlayControllerBrowserPDFTest
       public PDFExtensionTestBase {
  public:
   LensOverlayControllerBrowserPDFTest()
-      : base::test::WithFeatureOverride(chrome_pdf::features::kPdfOopif) {}
+      : base::test::WithFeatureOverride(chrome_pdf::features::kPdfOopif) {
+    tabs::TabFeatures::ReplaceTabFeaturesForTesting(
+        base::BindRepeating(&CreateTabFeatures));
+  }
 
   void SetUpOnMainThread() override {
     PDFExtensionTestBase::SetUpOnMainThread();
@@ -3672,6 +3679,33 @@ IN_PROC_BROWSER_TEST_P(LensOverlayControllerBrowserPDFTest,
   }));
 }
 
+IN_PROC_BROWSER_TEST_P(LensOverlayControllerBrowserPDFTest,
+                       PdfBytesExcludedInRequest) {
+  // Open the PDF document and wait for it to finish loading.
+  const GURL url = embedded_test_server()->GetURL(kPdfDocument);
+  content::RenderFrameHost* extension_host = LoadPdfGetExtensionHost(url);
+  ASSERT_TRUE(extension_host);
+
+  // State should start in off.
+  auto* controller = browser()
+                         ->tab_strip_model()
+                         ->GetActiveTab()
+                         ->tab_features()
+                         ->lens_overlay_controller();
+  ASSERT_EQ(controller->state(), State::kOff);
+
+  // Open the overlay.
+  controller->ShowUI(LensOverlayInvocationSource::kAppMenu);
+  ASSERT_EQ(controller->state(), State::kScreenshot);
+  ASSERT_TRUE(base::test::RunUntil(
+      [&]() { return controller->state() == State::kOverlay; }));
+
+  // Verify PDF bytes were excluded from the query.
+  auto* fake_query_controller = static_cast<LensOverlayQueryControllerFake*>(
+      controller->get_lens_overlay_query_controller_for_testing());
+  ASSERT_TRUE(fake_query_controller->last_sent_pdf_bytes_.empty());
+}
+
 // This test is wrapped in this BUILDFLAG block because the fallback region
 // search functionality will not be enabled if the flag is unset.
 #if BUILDFLAG(ENABLE_LENS_DESKTOP_GOOGLE_BRANDED_FEATURES)
@@ -3712,9 +3746,50 @@ IN_PROC_BROWSER_TEST_P(LensOverlayControllerBrowserPDFTest,
 }
 #endif  // BUILDFLAG(ENABLE_LENS_DESKTOP_GOOGLE_BRANDED_FEATURES)
 
+class LensOverlayControllerBrowserPDFContextualizationTest
+    : public LensOverlayControllerBrowserPDFTest {
+ public:
+  std::vector<base::test::FeatureRefAndParams> GetEnabledFeatures()
+      const override {
+    auto enabled = PDFExtensionTestBase::GetEnabledFeatures();
+    enabled.push_back(
+        {lens::features::kLensOverlay, {{"use-pdfs-as-context", "true"}}});
+    return enabled;
+  }
+};
+
+IN_PROC_BROWSER_TEST_P(LensOverlayControllerBrowserPDFContextualizationTest,
+                       PdfBytesIncludedInRequest) {
+  // Open the PDF document and wait for it to finish loading.
+  const GURL url = embedded_test_server()->GetURL(kPdfDocument);
+  content::RenderFrameHost* extension_host = LoadPdfGetExtensionHost(url);
+  ASSERT_TRUE(extension_host);
+
+  // State should start in off.
+  auto* controller = browser()
+                         ->tab_strip_model()
+                         ->GetActiveTab()
+                         ->tab_features()
+                         ->lens_overlay_controller();
+  ASSERT_EQ(controller->state(), State::kOff);
+
+  // Open the overlay.
+  controller->ShowUI(LensOverlayInvocationSource::kAppMenu);
+  ASSERT_EQ(controller->state(), State::kScreenshot);
+  ASSERT_TRUE(base::test::RunUntil(
+      [&]() { return controller->state() == State::kOverlay; }));
+
+  // Verify PDF bytes were included in the query.
+  auto* fake_query_controller = static_cast<LensOverlayQueryControllerFake*>(
+      controller->get_lens_overlay_query_controller_for_testing());
+  ASSERT_FALSE(fake_query_controller->last_sent_pdf_bytes_.empty());
+}
+
 // TODO(crbug.com/40268279): Stop testing both modes after OOPIF PDF viewer
 // launches.
 INSTANTIATE_FEATURE_OVERRIDE_TEST_SUITE(LensOverlayControllerBrowserPDFTest);
+INSTANTIATE_FEATURE_OVERRIDE_TEST_SUITE(
+    LensOverlayControllerBrowserPDFContextualizationTest);
 
 // Test with --enable-pixel-output-in-tests enabled, required to actually grab
 // screenshots for color extraction.

chrome/browser/ui/lens/lens_overlay_controller_interactive_uitest.cc

@@ -95,6 +95,7 @@ class LensOverlayQueryControllerFake : public lens::LensOverlayQueryController {
       std::optional<GURL> page_url,
       std::optional<std::string> page_title,
       std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes,
+      base::span<const uint8_t> pdf_bytes,
       float ui_scale_factor) override {
     // Send response for full image callback / HandleStartQueryResponse.
     base::SequencedTaskRunner::GetCurrentDefault()->PostTask(

chrome/browser/ui/lens/lens_overlay_query_controller.cc

@@ -75,6 +75,10 @@ constexpr char kOAuthConsumerName[] = "LensOverlayQueryController";
 constexpr char kStartTimeQueryParameter[] = "qsubts";
 constexpr char kGen204IdentifierQueryParameter[] = "plla";
 constexpr char kVisualSearchInteractionDataQueryParameterKey[] = "vsint";
+constexpr char kVisualInputTypeQueryParameterKey[] = "vit";
+// TODO(b/362997636): Video is temporary for prototyping. Needs to change once
+// the server is ready.
+constexpr char kPdfVisualInputTypeQueryParameterValue[] = "video";
 
 constexpr net::NetworkTrafficAnnotationTag kTrafficAnnotationTag =
     net::DefineNetworkTrafficAnnotation("lens_overlay", R"(
@@ -220,11 +224,13 @@ void LensOverlayQueryController::StartQueryFlow(
     std::optional<GURL> page_url,
     std::optional<std::string> page_title,
     std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes,
+    base::span<const uint8_t> pdf_bytes,
     float ui_scale_factor) {
   original_screenshot_ = screenshot;
   page_url_ = page_url;
   page_title_ = page_title;
   significant_region_boxes_ = std::move(significant_region_boxes);
+  pdf_bytes_ = pdf_bytes;
   ui_scale_factor_ = ui_scale_factor;
   gen204_id_ = base::RandUint64();
 
@@ -356,6 +362,16 @@ void LensOverlayQueryController::FetchFullImageRequest(
       request_context);
   request.mutable_objects_request()->mutable_image_data()->CopyFrom(image_data);
 
+  // The PDF bytes are optional, so if they were included in the StartQueryFlow,
+  // included them in the request.
+  if (!pdf_bytes_.empty()) {
+    lens::Payload payload;
+    payload.mutable_content_data()->assign(pdf_bytes_.begin(),
+                                           pdf_bytes_.end());
+    payload.set_content_type("application/pdf");
+    request.mutable_objects_request()->mutable_payload()->CopyFrom(payload);
+  }
+
   int64_t query_start_time_ms =
       base::Time::Now().InMillisecondsSinceUnixEpoch();
   latest_full_image_sequence_id_ = request_id->sequence_id();
@@ -573,8 +589,7 @@ void LensOverlayQueryController::SendMultimodalRequest(
   if (base::TrimWhitespaceASCII(query_text, base::TRIM_ALL).empty()) {
     return;
   }
-  SendInteraction(/*region=*/std::move(region),
-                  /*query_text=*/std::make_optional<std::string>(query_text),
+  SendInteraction(/*region=*/std::move(region), query_text,
                   /*object_id=*/std::nullopt, multimodal_selection_type,
                   additional_search_query_params, region_bytes);
 }
@@ -586,6 +601,21 @@ void LensOverlayQueryController::SendTextOnlyQuery(
   // Increment the request counter to cancel previously issued fetches.
   request_counter_++;
 
+  // If PDF bytes exist on a text only query, contextualize the query via a Lens
+  // request, instead of going straight through GWS.
+  if (!pdf_bytes_.empty()) {
+    // Include the vit to get contextualized results.
+    additional_search_query_params.insert(
+        {kVisualInputTypeQueryParameterKey,
+         kPdfVisualInputTypeQueryParameterValue});
+
+    // TODO(b/362816047): Send the correct selection type once it is ready.
+    SendInteraction(/*region=*/nullptr, query_text,
+                    /*object_id=*/std::nullopt, lens::UNKNOWN_SELECTION_TYPE,
+                    additional_search_query_params, std::nullopt);
+    return;
+  }
+
   // Add the start time to the query params now, so that any additional
   // client processing time is included.
   additional_search_query_params =
@@ -749,6 +779,15 @@ LensOverlayQueryController::CreateInteractionRequest(
     interaction_request_metadata.mutable_selection_metadata()
         ->mutable_object()
         ->set_object_id(*object_id);
+  } else if (query_text.has_value()) {
+    // If there is only `query_text`, this is a contextual flow.
+    // TOOD(b/362816047): Send correct LensOverlayInteractionRequestMetadata,
+    // once the server is ready for it.
+    interaction_request_metadata.set_type(
+        lens::LensOverlayInteractionRequestMetadata::UNKNOWN);
+    interaction_request_metadata.mutable_query_metadata()
+        ->mutable_text_query()
+        ->set_query(*query_text);
   } else {
     // There should be a region or an object id in the request.
     NOTREACHED_IN_MIGRATION();

chrome/browser/ui/lens/lens_overlay_query_controller.h

@@ -5,6 +5,7 @@
 #ifndef CHROME_BROWSER_UI_LENS_LENS_OVERLAY_QUERY_CONTROLLER_H_
 #define CHROME_BROWSER_UI_LENS_LENS_OVERLAY_QUERY_CONTROLLER_H_
 
+#include "base/containers/span.h"
 #include "base/functional/callback.h"
 #include "base/task/cancelable_task_tracker.h"
 #include "chrome/browser/lens/core/mojom/lens.mojom.h"
@@ -79,6 +80,7 @@ class LensOverlayQueryController {
       std::optional<GURL> page_url,
       std::optional<std::string> page_title,
       std::vector<lens::mojom::CenterRotatedBoxPtr> significant_region_boxes,
+      base::span<const uint8_t> pdf_bytes,
       float ui_scale_factor);
 
   // Clears the state and resets stored values.
@@ -364,6 +366,10 @@ class LensOverlayQueryController {
 
   const raw_ptr<Profile> profile_;
 
+  // PDF bytes the user is viewing. Owned by LensOverlayController. Will be
+  // empty if no PDF bytes to the underlying page exists.
+  base::span<const uint8_t> pdf_bytes_;
+
   // The request counter, used to make sure requests are not sent out of
   // order.
   int request_counter_ = 0;

chrome/browser/ui/lens/lens_overlay_query_controller_unittest.cc

@@ -63,6 +63,9 @@ constexpr char kVisualSearchInteractionDataQueryParameterKey[] = "vsint";
 // Query parameter for the request id.
 inline constexpr char kRequestIdParameterKey[] = "vsrid";
 
+// Query parameter for the visual input type.
+inline constexpr char kVisualInputTypeParameterKey[] = "vit";
+
 // The encoded search context for the test page and title.
 constexpr char kTestEncodedSearchContext[] =
     "ChdodHRwczovL3d3dy5nb29nbGUuY29tLxIKUGFnZSBUaXRsZQ";
@@ -266,7 +269,8 @@ TEST_F(LensOverlayQueryControllerTest, FetchInitialQuery_ReturnsResponse) {
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
 
   task_environment_.RunUntilIdle();
   query_controller.EndQuery();
@@ -332,7 +336,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   auto region = lens::mojom::CenterRotatedBox::New();
@@ -437,7 +442,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       viewport_bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   SkBitmap region_bitmap = CreateNonEmptyBitmap(100, 100);
@@ -558,7 +564,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   auto region = lens::mojom::CenterRotatedBox::New();
@@ -665,7 +672,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   query_controller.SendTextOnlyQuery("", TextOnlyQueryType::kLensTextSelection,
@@ -693,6 +701,97 @@ TEST_F(LensOverlayQueryControllerTest,
   ASSERT_EQ(query_controller.num_gen204_pings_sent_, 0);
 }
 
+TEST_F(LensOverlayQueryControllerTest,
+       FetchTextOnlyInteractionWithPdf_ReturnsResponse) {
+  base::test::TestFuture<std::vector<lens::mojom::OverlayObjectPtr>,
+                         lens::mojom::TextPtr, bool>
+      full_image_response_future;
+  base::test::TestFuture<lens::proto::LensOverlayUrlResponse>
+      url_response_future;
+  base::test::TestFuture<lens::proto::LensOverlayInteractionResponse>
+      interaction_data_response_future;
+  base::test::TestFuture<const std::string&> thumbnail_created_future;
+  LensOverlayQueryControllerMock query_controller(
+      full_image_response_future.GetRepeatingCallback(),
+      url_response_future.GetRepeatingCallback(),
+      interaction_data_response_future.GetRepeatingCallback(),
+      thumbnail_created_future.GetRepeatingCallback(),
+      profile()->GetVariationsClient(),
+      IdentityManagerFactory::GetForProfile(profile()), profile(),
+      lens::LensOverlayInvocationSource::kAppMenu,
+      /*use_dark_mode=*/false);
+  query_controller.fake_objects_response_.mutable_cluster_info()
+      ->set_server_session_id(kTestServerSessionId);
+  query_controller.fake_interaction_response_.set_encoded_response(
+      kTestSuggestSignals);
+  SkBitmap bitmap = CreateNonEmptyBitmap(100, 100);
+  std::map<std::string, std::string> additional_search_query_params;
+  std::vector<uint8_t> fake_pdf_bytes({1, 2, 3, 4});
+  query_controller.StartQueryFlow(
+      bitmap, std::make_optional<GURL>(kTestPageUrl),
+      std::make_optional<std::string>(kTestPageTitle),
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(), fake_pdf_bytes, 0);
+  task_environment_.RunUntilIdle();
+  query_controller.SendTextOnlyQuery(kTestQueryText,
+                                     TextOnlyQueryType::kLensTextSelection,
+                                     additional_search_query_params);
+  task_environment_.RunUntilIdle();
+  query_controller.EndQuery();
+
+  // Check initial fetch objects request is correct.
+  ASSERT_TRUE(full_image_response_future.IsReady());
+  ASSERT_EQ(query_controller.sent_objects_request_.image_data()
+                .image_metadata()
+                .width(),
+            100);
+  ASSERT_EQ(query_controller.sent_objects_request_.image_data()
+                .image_metadata()
+                .height(),
+            100);
+  ASSERT_FALSE(
+      query_controller.sent_objects_request_.payload().content_data().empty());
+  ASSERT_EQ(query_controller.sent_objects_request_.payload().content_type(),
+            "application/pdf");
+
+  // Check interaction request is correct.
+  ASSERT_TRUE(interaction_data_response_future.IsReady());
+  ASSERT_EQ(query_controller.sent_interaction_request_.request_context()
+                .request_id()
+                .sequence_id(),
+            2);
+  ASSERT_EQ(
+      query_controller.sent_interaction_request_.interaction_request_metadata()
+          .type(),
+      lens::LensOverlayInteractionRequestMetadata::UNKNOWN);
+  ASSERT_EQ(
+      query_controller.sent_interaction_request_.interaction_request_metadata()
+          .query_metadata()
+          .text_query()
+          .query(),
+      kTestQueryText);
+  ASSERT_FALSE(
+      query_controller.sent_interaction_request_.interaction_request_metadata()
+          .has_selection_metadata());
+
+  // Check search URL is correct.
+  ASSERT_TRUE(url_response_future.IsReady());
+  std::string unused_start_time;
+  bool has_start_time =
+      net::GetValueForKeyInQuery(GURL(url_response_future.Get().url()),
+                                 kStartTimeQueryParam, &unused_start_time);
+  std::string visual_input_type;
+  bool has_visual_input_type = net::GetValueForKeyInQuery(
+      GURL(url_response_future.Get().url()), kVisualInputTypeParameterKey,
+      &visual_input_type);
+  ASSERT_EQ(GetSelectionTypeFromUrl(url_response_future.Get().url()),
+            lens::UNKNOWN_SELECTION_TYPE);
+  ASSERT_TRUE(has_start_time);
+  ASSERT_TRUE(has_visual_input_type);
+  ASSERT_EQ(visual_input_type, "video");
+  ASSERT_EQ(query_controller.num_gen204_pings_sent_, 1);
+  ASSERT_TRUE(url_response_future.Get().has_url());
+}
+
 TEST_F(LensOverlayQueryControllerTest,
        FetchInteraction_StartsNewQueryFlowAfterTimeout) {
   base::test::TestFuture<std::vector<lens::mojom::OverlayObjectPtr>,
@@ -726,7 +825,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   ASSERT_TRUE(full_image_response_future.IsReady());
@@ -783,7 +883,8 @@ TEST_F(LensOverlayQueryControllerTest,
   query_controller.StartQueryFlow(
       bitmap, std::make_optional<GURL>(kTestPageUrl),
       std::make_optional<std::string>(kTestPageTitle),
-      std::vector<lens::mojom::CenterRotatedBoxPtr>(), 0);
+      std::vector<lens::mojom::CenterRotatedBoxPtr>(),
+      /*pdf_bytes=*/{}, 0);
   task_environment_.RunUntilIdle();
 
   ASSERT_TRUE(full_image_response_future.IsReady());

components/lens/lens_features.cc

@@ -178,6 +178,9 @@ constexpr base::FeatureParam<bool>
     kUseSearchContextForMultimodalLensOverlayRequests{
         &kLensOverlay, "use-search-context-for-multimodal-requests", false};
 
+constexpr base::FeatureParam<bool> kUsePdfsAsContext{
+    &kLensOverlay, "use-pdfs-as-context", false};
+
 constexpr base::FeatureParam<int> kLensOverlayTapRegionHeight{
     &kLensOverlay, "tap-region-height", 300};
 constexpr base::FeatureParam<int> kLensOverlayTapRegionWidth{
@@ -447,6 +450,10 @@ bool UseSearchContextForMultimodalLensOverlayRequests() {
   return kUseSearchContextForMultimodalLensOverlayRequests.Get();
 }
 
+bool UsePdfsAsContext() {
+  return kUsePdfsAsContext.Get();
+}
+
 int GetLensOverlayVerticalTextMargin() {
   return kLensOverlayVerticalTextMargin.Get();
 }

components/lens/lens_features.h

@@ -333,6 +333,11 @@ extern bool UseSearchContextForTextOnlyLensOverlayRequests();
 COMPONENT_EXPORT(LENS_FEATURES)
 extern bool UseSearchContextForMultimodalLensOverlayRequests();
 
+// Returns whether to include PDFs from the underlying page in the request to be
+// used as page context.
+COMPONENT_EXPORT(LENS_FEATURES)
+extern bool UsePdfsAsContext();
+
 // Returns the margin in pixels to add to the top and bottom of word bounding
 // boxes.
 COMPONENT_EXPORT(LENS_FEATURES)

components/pdf/browser/pdf_document_helper.cc

@@ -15,6 +15,7 @@
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/referrer_type_converters.h"
+#include "pdf/mojom/pdf.mojom.h"
 #include "pdf/pdf_features.h"
 #include "ui/base/pointer/touch_editing_controller.h"
 #include "ui/base/ui_base_types.h"
@@ -201,6 +202,15 @@ void PDFDocumentHelper::SelectBetweenCoordinates(const gfx::PointF& base,
                                          ConvertFromRoot(extent));
 }
 
+void PDFDocumentHelper::GetPdfBytes(
+    pdf::mojom::PdfListener::GetPdfBytesCallback callback) {
+  if (!remote_pdf_client_) {
+    std::move(callback).Run(std::vector<uint8_t>());
+    return;
+  }
+  remote_pdf_client_->GetPdfBytes(std::move(callback));
+}
+
 void PDFDocumentHelper::OnSelectionEvent(ui::SelectionEventType event) {
   // Should be handled by `TouchSelectionControllerClientAura`.
   NOTREACHED_IN_MIGRATION();

components/pdf/browser/pdf_document_helper.h

@@ -87,6 +87,8 @@ class PDFDocumentHelper
                         int32_t right_height) override;
   void SetPluginCanSave(bool can_save) override;
 
+  void GetPdfBytes(pdf::mojom::PdfListener::GetPdfBytesCallback callback);
+
  private:
   friend class content::DocumentUserData<PDFDocumentHelper>;
 

components/pdf/browser/pdf_document_helper_browsertest.cc

@@ -41,6 +41,7 @@ class FakePdfListener : public pdf::mojom::PdfListener {
               SetSelectionBounds,
               (const gfx::PointF&, const gfx::PointF&),
               (override));
+  MOCK_METHOD(void, GetPdfBytes, (GetPdfBytesCallback callback), (override));
 };
 
 class TestPDFDocumentHelperClient : public PDFDocumentHelperClient {

pdf/mojom/pdf.mojom

@@ -18,6 +18,9 @@ interface PdfListener {
   // Sets the selection to be between |base| and |extent|. The |extent| will
   // be moved if the selection is modified.
   SetSelectionBounds(gfx.mojom.PointF base, gfx.mojom.PointF extent);
+
+  // Get PDF bytes.
+  GetPdfBytes() => (array<uint8> bytes);
 };
 
 // Browser-side interface used by PDF renderers.
@@ -35,8 +38,10 @@ interface PdfHost {
 
   // Notifies the embedder of the top-left and bottom-right coordinates of the
   // current selection.
-  SelectionChanged(gfx.mojom.PointF left, int32 left_height,
-                   gfx.mojom.PointF right, int32 right_height);
+  SelectionChanged(gfx.mojom.PointF left,
+                   int32 left_height,
+                   gfx.mojom.PointF right,
+                   int32 right_height);
 
   // Notifies the embedder know the plugin can handle save commands internally.
   SetPluginCanSave(bool can_save);

pdf/pdf_view_web_plugin.cc

@@ -1362,6 +1362,10 @@ void PdfViewWebPlugin::SetSelectionBounds(const gfx::PointF& base,
                               FrameToPdfCoordinates(extent));
 }
 
+void PdfViewWebPlugin::GetPdfBytes(GetPdfBytesCallback callback) {
+  std::move(callback).Run(engine_->GetSaveData());
+}
+
 bool PdfViewWebPlugin::IsValid() const {
   return client_->HasFrame();
 }

pdf/pdf_view_web_plugin.h

@@ -369,6 +369,7 @@ class PdfViewWebPlugin final : public PDFiumEngineClient,
   void MoveRangeSelectionExtent(const gfx::PointF& extent) override;
   void SetSelectionBounds(const gfx::PointF& base,
                           const gfx::PointF& extent) override;
+  void GetPdfBytes(GetPdfBytesCallback callback) override;
 
   // UrlLoader::Client:
   bool IsValid() const override;