Reland "Extract high quality images from PDF only when they are needed for OCR."

This is a reland of commit 5208eafb23.
Fixed the build issue by moving the definition of
`TestPdfAccessibilityTree` out of BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
and wrapping only necessary class functions and members with the
buildflag.

Original change's description:
> Extract high quality images from PDF only when they are needed for OCR.
>
> 32-bit bitmaps with highest available quality are extracted from PDF
> files when they are loaded, so that they would be sent later to OCR
> service. To avoid the memory overhead of this process, this CL
> postpones image extraction to the time they are sent to OCR service,
> and destroys the extracted images immediately after that.
>
> AX-Relnotes: n/a
> Bug: 1471392
> Change-Id: Id337edf693d8d4a4ddd1a56d814a0d1f0e1ac5e4
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4754282
> Auto-Submit: Ramin Halavati <rhalavati@chromium.org>
> Reviewed-by: Kyungjun Lee <kyungjunlee@google.com>
> Commit-Queue: Ramin Halavati <rhalavati@chromium.org>
> Commit-Queue: Lei Zhang <thestig@chromium.org>
> Reviewed-by: Lei Zhang <thestig@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#1182463}

Bug: 1471392
Change-Id: I1762da4b5d5895ce112f62d0ca99c23236a6135d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4775300
Reviewed-by: Lei Zhang <thestig@chromium.org>
Reviewed-by: Nektarios Paisios <nektar@chromium.org>
Commit-Queue: Kyungjun Lee <kyungjunlee@google.com>
Cr-Commit-Position: refs/heads/main@{#1182838}

components/pdf/renderer/DEPS

@@ -12,6 +12,7 @@ include_rules = [
   "+pdf/mojom/pdf.mojom.h",
   "+pdf/pdf_accessibility_action_handler.h",
   "+pdf/pdf_accessibility_data_handler.h",
+  "+pdf/pdf_accessibility_image_fetcher.h",
   "+pdf/pdf_features.h",
   "+pdf/pdf_view_web_plugin.h",
   "+printing/buildflags/buildflags.h",

components/pdf/renderer/pdf_accessibility_tree.cc

@@ -25,6 +25,7 @@
 #include "content/public/renderer/render_frame.h"
 #include "content/public/renderer/render_thread.h"
 #include "pdf/pdf_accessibility_action_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_features.h"
 #include "third_party/blink/public/strings/grit/blink_accessibility_strings.h"
 #include "ui/accessibility/ax_enums.mojom.h"
@@ -57,10 +58,12 @@ using PdfOcrRequest = PdfAccessibilityTree::PdfOcrRequest;
 
 PdfOcrRequest::PdfOcrRequest(const ui::AXNodeID& image_node_id,
                              const chrome_pdf::AccessibilityImageInfo& image,
-                             const ui::AXNodeID& parent_node_id)
+                             const ui::AXNodeID& parent_node_id,
+                             uint32_t page_index)
     : image_node_id(image_node_id),
       image(image),
-      parent_node_id(parent_node_id) {}
+      parent_node_id(parent_node_id),
+      page_index(page_index) {}
 
 //
 // PdfOcrService
@@ -68,10 +71,13 @@ PdfOcrRequest::PdfOcrRequest(const ui::AXNodeID& image_node_id,
 
 using PdfOcrService = PdfAccessibilityTree::PdfOcrService;
 
-PdfOcrService::PdfOcrService(content::RenderFrame& render_frame,
-                             uint32_t page_count,
-                             OnOcrDataReceivedCallback callback)
-    : remaining_page_count_(page_count),
+PdfOcrService::PdfOcrService(
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher,
+    content::RenderFrame& render_frame,
+    uint32_t page_count,
+    OnOcrDataReceivedCallback callback)
+    : image_fetcher_(image_fetcher),
+      remaining_page_count_(page_count),
       on_ocr_data_received_callback_(std::move(callback)) {
   CHECK(features::IsPdfOcrEnabled());
   render_frame.GetBrowserInterfaceBroker()->GetInterface(
@@ -124,12 +130,21 @@ void PdfOcrService::OcrNextImage() {
   if (all_requests_.empty()) {
     return;
   }
-  const PdfOcrRequest request = all_requests_.front();
+  PdfOcrRequest request = all_requests_.front();
   all_requests_.pop();
+
+  SkBitmap bitmap = image_fetcher_->GetImageForOcr(
+      request.page_index, request.image.page_object_index);
+  request.image_pixel_size = gfx::SizeF(bitmap.width(), bitmap.height());
+  if (bitmap.drawsNothing()) {
+    ReceiveOcrResultsForImage(std::move(request), ui::AXTreeUpdate());
+    return;
+  }
+
   screen_ai_annotator_->PerformOcrAndReturnAXTreeUpdate(
-      request.image.image_data,
+      std::move(bitmap),
       base::BindOnce(&PdfOcrService::ReceiveOcrResultsForImage,
-                     weak_ptr_factory_.GetWeakPtr(), request));
+                     weak_ptr_factory_.GetWeakPtr(), std::move(request)));
 
   base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
                                 PdfOcrRequestStatus::kRequested);
@@ -506,14 +521,11 @@ std::unique_ptr<ui::AXNodeData> CreateStatusNodeWrapper(
   return node_wrapper;
 }
 
-gfx::Transform MakeTransformForImage(
-    const chrome_pdf::AccessibilityImageInfo& image) {
+gfx::Transform MakeTransformForImage(const gfx::RectF image_screen_size,
+                                     const gfx::SizeF image_pixel_size) {
   // Nodes created with OCR results from the image will be misaligned on screen
   // if `image_screen_size` is different from `image_pixel_size`. To address
   // this misalignment issue, an additional transform needs to be created.
-  const gfx::RectF& image_screen_size = image.bounds;
-  const gfx::RectF image_pixel_size =
-      gfx::RectF(image.image_data.width(), image.image_data.height());
   CHECK(!image_pixel_size.IsEmpty());
 
   gfx::Transform transform;
@@ -1331,9 +1343,9 @@ class PdfAccessibilityTreeBuilder {
       ui::AXNodeData* image_node = CreateImageNode(images_[i]);
       para_node->child_ids.push_back(image_node->id);
 #if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
-      if (!has_accessible_text_ && ocr_available &&
-          !images_[i].image_data.drawsNothing()) {
-        ocr_requests.emplace(image_node->id, images_[i], para_node->id);
+      if (!has_accessible_text_ && ocr_available) {
+        ocr_requests.emplace(image_node->id, images_[i], para_node->id,
+                             page_index_);
       }
 #endif
     }
@@ -1409,12 +1421,15 @@ class PdfAccessibilityTreeBuilder {
 
 PdfAccessibilityTree::PdfAccessibilityTree(
     content::RenderFrame* render_frame,
-    chrome_pdf::PdfAccessibilityActionHandler* action_handler)
+    chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher)
     : content::RenderFrameObserver(render_frame),
       render_frame_(render_frame),
-      action_handler_(action_handler) {
+      action_handler_(action_handler),
+      image_fetcher_(image_fetcher) {
   DCHECK(render_frame);
   DCHECK(action_handler_);
+  DCHECK(image_fetcher_);
   MaybeHandleAccessibilityChange(/*always_load_or_reload_accessibility=*/false);
 }
 
@@ -2107,14 +2122,17 @@ void PdfAccessibilityTree::OnOcrDataReceived(
     // would be more convenient and less complex if an `ui::AXTree` was never
     // constructed and if the `ui::AXTreeSource` was able to use the collection
     // of `nodes_` directly.
-    base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
-                                  PdfOcrRequestStatus::kPerformed);
 
     if (tree_update.nodes.empty()) {
       VLOG(1) << "Empty OCR data received.";
+      // TODO(crbug.com/1471392): Create an empty update and continue. This can
+      // happen if OCR returns an empty result, or the image draws nothing.
       return;
     }
 
+    base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
+                                  PdfOcrRequestStatus::kPerformed);
+
     // Update the flag if OCR extracted text from any images. This flag will be
     // used to update the status node to notify users of it.
     was_text_converted_from_image_ = true;
@@ -2145,7 +2163,8 @@ void PdfAccessibilityTree::OnOcrDataReceived(
     // transform, nodes created from OCR results will have misaligned bounding
     // boxes. This transform will be applied to all nodes from OCR results
     // below.
-    gfx::Transform transform = MakeTransformForImage(ocr_request.image);
+    gfx::Transform transform = MakeTransformForImage(
+        ocr_request.image.bounds, ocr_request.image_pixel_size);
 
     // Count each detected language and find out the most detected language in
     // OCR result. Then record the most detected language in UMA.
@@ -2231,7 +2250,7 @@ void PdfAccessibilityTree::OnOcrDataReceived(
 void PdfAccessibilityTree::CreateOcrService() {
   VLOG(2) << "Creating OCR service.";
   ocr_service_ = std::make_unique<PdfOcrService>(
-      *render_frame_, page_count_,
+      image_fetcher_, *render_frame_, page_count_,
       base::BindRepeating(&PdfAccessibilityTree::OnOcrDataReceived,
                           weak_ptr_factory_.GetWeakPtr()));
 }

components/pdf/renderer/pdf_accessibility_tree.h

@@ -35,6 +35,7 @@
 namespace chrome_pdf {
 
 class PdfAccessibilityActionHandler;
+class PdfAccessibilityImageFetcher;
 
 }  // namespace chrome_pdf
 
@@ -72,14 +73,19 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
   struct PdfOcrRequest {
     PdfOcrRequest(const ui::AXNodeID& image_node_id,
                   const chrome_pdf::AccessibilityImageInfo& image,
-                  const ui::AXNodeID& parent_node_id);
+                  const ui::AXNodeID& parent_node_id,
+                  uint32_t page_index);
 
     const ui::AXNodeID image_node_id;
     const chrome_pdf::AccessibilityImageInfo image;
     const ui::AXNodeID parent_node_id;
+    const uint32_t page_index;
     // This boolean indicates which request corresponds to the last image on
     // each page.
     bool is_last_on_page = false;
+
+    // This field is set after the image is extracted from PDF.
+    gfx::SizeF image_pixel_size;
   };
 
   // Manages the connection to the OCR Service via Mojo, and ensures that
@@ -90,7 +96,8 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
         std::vector<PdfOcrRequest> ocr_requests,
         std::vector<ui::AXTreeUpdate> tree_updates)>;
 
-    PdfOcrService(content::RenderFrame& render_frame,
+    PdfOcrService(chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher,
+                  content::RenderFrame& render_frame,
                   uint32_t page_count,
                   OnOcrDataReceivedCallback callback);
 
@@ -124,6 +131,9 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
     void ReceiveOcrResultsForImage(PdfOcrRequest request,
                                    const ui::AXTreeUpdate& tree_update);
 
+    // `image_fetcher_` owns `this`.
+    chrome_pdf::PdfAccessibilityImageFetcher* const image_fetcher_;
+
     uint32_t remaining_page_count_;
     // True if there are pending OCR requests. Used to determine if `OcrPage`
     // should call `OcrNextImage` or if the next call to
@@ -149,7 +159,8 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
 
   PdfAccessibilityTree(
       content::RenderFrame* render_frame,
-      chrome_pdf::PdfAccessibilityActionHandler* action_handler);
+      chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+      chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher);
   ~PdfAccessibilityTree() override;
 
   static bool IsDataFromPluginValid(
@@ -305,6 +316,7 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
 
   // Unowned. Must outlive `this`.
   chrome_pdf::PdfAccessibilityActionHandler* const action_handler_;
+  chrome_pdf::PdfAccessibilityImageFetcher* const image_fetcher_;
 
   // `zoom_` signifies the zoom level set in for the browser content.
   // `scale_` signifies the scale level set by user. Scale is applied

components/pdf/renderer/pdf_view_web_plugin_client.cc

@@ -255,8 +255,10 @@ void PdfViewWebPluginClient::RecordComputedAction(const std::string& action) {
 
 std::unique_ptr<chrome_pdf::PdfAccessibilityDataHandler>
 PdfViewWebPluginClient::CreateAccessibilityDataHandler(
-    chrome_pdf::PdfAccessibilityActionHandler* action_handler) {
-  return std::make_unique<PdfAccessibilityTree>(render_frame_, action_handler);
+    chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher) {
+  return std::make_unique<PdfAccessibilityTree>(render_frame_, action_handler,
+                                                image_fetcher);
 }
 
 }  // namespace pdf

components/pdf/renderer/pdf_view_web_plugin_client.h

@@ -78,7 +78,8 @@ class PdfViewWebPluginClient : public chrome_pdf::PdfViewWebPlugin::Client {
   void RecordComputedAction(const std::string& action) override;
   std::unique_ptr<chrome_pdf::PdfAccessibilityDataHandler>
   CreateAccessibilityDataHandler(
-      chrome_pdf::PdfAccessibilityActionHandler* action_handler) override;
+      chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+      chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher) override;
 
  private:
   blink::WebLocalFrame* GetFrame() const;

pdf/BUILD.gn

@@ -217,6 +217,7 @@ if (enable_pdf) {
       "accessibility_structs.h",
       "pdf_accessibility_action_handler.h",
       "pdf_accessibility_data_handler.h",
+      "pdf_accessibility_image_fetcher.h",
     ]
 
     configs += [ ":strict" ]

pdf/accessibility_structs.cc

@@ -77,11 +77,11 @@ AccessibilityImageInfo::AccessibilityImageInfo() = default;
 AccessibilityImageInfo::AccessibilityImageInfo(const std::string& alt_text,
                                                uint32_t text_run_index,
                                                const gfx::RectF& bounds,
-                                               const SkBitmap& image_data)
+                                               int32_t page_object_index)
     : alt_text(alt_text),
       text_run_index(text_run_index),
       bounds(bounds),
-      image_data(image_data) {}
+      page_object_index(page_object_index) {}
 
 AccessibilityImageInfo::AccessibilityImageInfo(
     const AccessibilityImageInfo& other) = default;

pdf/accessibility_structs.h

@@ -132,7 +132,7 @@ struct AccessibilityImageInfo {
   AccessibilityImageInfo(const std::string& alt_text,
                          uint32_t text_run_index,
                          const gfx::RectF& bounds,
-                         const SkBitmap& image_data);
+                         int32_t page_object_index);
   AccessibilityImageInfo(const AccessibilityImageInfo& other);
   ~AccessibilityImageInfo();
 
@@ -147,9 +147,8 @@ struct AccessibilityImageInfo {
   // Bounding box of the image.
   gfx::RectF bounds;
 
-  // Only populated if `alt_text` is empty or unavailable, and if the user has
-  // requested that the OCR service tag the PDF so that it is made accessible.
-  SkBitmap image_data;
+  // Index of the image object in its page.
+  int32_t page_object_index;
 };
 
 struct AccessibilityHighlightInfo {

pdf/pdf_accessibility_image_fetcher.h

@@ -0,0 +1,22 @@
+// Copyright 2023 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_
+#define PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_
+
+class SkBitmap;
+
+namespace chrome_pdf {
+
+class PdfAccessibilityImageFetcher {
+ public:
+  virtual ~PdfAccessibilityImageFetcher() = default;
+  // Fetches the image as a 32-bit bitmap for OCR.
+  virtual SkBitmap GetImageForOcr(int32_t page_index,
+                                  int32_t page_object_index) = 0;
+};
+
+}  // namespace chrome_pdf
+
+#endif  // PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_

pdf/pdf_engine.h

@@ -408,6 +408,8 @@ class PDFEngine {
   virtual std::vector<AccessibilityImageInfo> GetImageInfo(
       int page_index,
       uint32_t text_run_count) = 0;
+  // Returns the image as a 32-bit bitmap format for OCR.
+  virtual SkBitmap GetImageForOcr(int page_index, int image_index) = 0;
   // For all the highlights in page `page_index`, get their underlying text
   // ranges and bounding boxes.
   virtual std::vector<AccessibilityHighlightInfo> GetHighlightInfo(

pdf/pdf_view_web_plugin.cc

@@ -274,7 +274,8 @@ std::unique_ptr<PDFiumEngine> PdfViewWebPlugin::Client::CreateEngine(
 
 std::unique_ptr<PdfAccessibilityDataHandler>
 PdfViewWebPlugin::Client::CreateAccessibilityDataHandler(
-    PdfAccessibilityActionHandler* action_handler) {
+    PdfAccessibilityActionHandler* action_handler,
+    PdfAccessibilityImageFetcher* image_fetcher) {
   return nullptr;
 }
 
@@ -286,7 +287,7 @@ PdfViewWebPlugin::PdfViewWebPlugin(
       pdf_service_(std::move(pdf_service)),
       initial_params_(params),
       pdf_accessibility_data_handler_(
-          client_->CreateAccessibilityDataHandler(this)) {
+          client_->CreateAccessibilityDataHandler(this, this)) {
   DCHECK(pdf_service_);
   pdf_service_->SetListener(listener_receiver_.BindNewPipeAndPassRemote());
 }
@@ -1929,6 +1930,11 @@ void PdfViewWebPlugin::EnableAccessibility() {
   LoadOrReloadAccessibility();
 }
 
+SkBitmap PdfViewWebPlugin::GetImageForOcr(int32_t page_index,
+                                          int32_t page_object_index) {
+  return engine_->GetImageForOcr(page_index, page_object_index);
+}
+
 void PdfViewWebPlugin::HandleAccessibilityAction(
     const AccessibilityActionData& action_data) {
   engine_->HandleAccessibilityAction(action_data);

pdf/pdf_view_web_plugin.h

@@ -27,6 +27,7 @@
 #include "pdf/mojom/pdf.mojom.h"
 #include "pdf/paint_manager.h"
 #include "pdf/pdf_accessibility_action_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_engine.h"
 #include "pdf/pdfium/pdfium_form_filler.h"
 #include "pdf/post_message_receiver.h"
@@ -82,6 +83,7 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
                                public PostMessageReceiver::Client,
                                public PaintManager::Client,
                                public PdfAccessibilityActionHandler,
+                               public PdfAccessibilityImageFetcher,
                                public PreviewModeClient::Client {
  public:
   // Do not save files larger than 100 MB. This cap should be kept in sync with
@@ -219,7 +221,8 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
     // client.
     virtual std::unique_ptr<PdfAccessibilityDataHandler>
     CreateAccessibilityDataHandler(
-        PdfAccessibilityActionHandler* action_handler);
+        PdfAccessibilityActionHandler* action_handler,
+        PdfAccessibilityImageFetcher* image_fetcher);
   };
 
   PdfViewWebPlugin(std::unique_ptr<Client> client,
@@ -381,6 +384,10 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
       const AccessibilityActionData& action_data) override;
   void LoadOrReloadAccessibility() override;
 
+  // PdfAccessibilityImageFetcher:
+  SkBitmap GetImageForOcr(int32_t page_index,
+                          int32_t page_object_index) override;
+
   // PreviewModeClient::Client:
   void PreviewDocumentLoadComplete() override;
   void PreviewDocumentLoadFailed() override;

pdf/pdf_view_web_plugin_unittest.cc

@@ -37,6 +37,7 @@
 #include "pdf/mojom/pdf.mojom.h"
 #include "pdf/paint_ready_rect.h"
 #include "pdf/pdf_accessibility_data_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_features.h"
 #include "pdf/test/mock_web_associated_url_loader.h"
 #include "pdf/test/test_helpers.h"
@@ -308,7 +309,7 @@ class FakePdfViewWebPluginClient : public PdfViewWebPlugin::Client {
 
   MOCK_METHOD(std::unique_ptr<PdfAccessibilityDataHandler>,
               CreateAccessibilityDataHandler,
-              (PdfAccessibilityActionHandler*),
+              (PdfAccessibilityActionHandler*, PdfAccessibilityImageFetcher*),
               (override));
 };
 

pdf/pdfium/pdfium_engine.cc

@@ -2618,6 +2618,11 @@ std::vector<AccessibilityImageInfo> PDFiumEngine::GetImageInfo(
   return pages_[page_index]->GetImageInfo(text_run_count);
 }
 
+SkBitmap PDFiumEngine::GetImageForOcr(int page_index, int image_index) {
+  DCHECK(PageIndexInBounds(page_index));
+  return pages_[page_index]->GetImageForOcr(image_index);
+}
+
 std::vector<AccessibilityHighlightInfo> PDFiumEngine::GetHighlightInfo(
     int page_index,
     const std::vector<AccessibilityTextRunInfo>& text_runs) {

pdf/pdfium/pdfium_engine.h

@@ -158,6 +158,7 @@ class PDFiumEngine : public PDFEngine,
   std::vector<AccessibilityImageInfo> GetImageInfo(
       int page_index,
       uint32_t text_run_count) override;
+  SkBitmap GetImageForOcr(int page_index, int image_index) override;
   std::vector<AccessibilityHighlightInfo> GetHighlightInfo(
       int page_index,
       const std::vector<AccessibilityTextRunInfo>& text_runs) override;

pdf/pdfium/pdfium_page.cc

@@ -770,12 +770,84 @@ std::vector<AccessibilityImageInfo> PDFiumPage::GetImageInfo(
     cur_info.bounds =
         gfx::RectF(image.bounding_rect.x(), image.bounding_rect.y(),
                    image.bounding_rect.width(), image.bounding_rect.height());
-    cur_info.image_data = image.image_data;
+    cur_info.page_object_index = image.page_object_index;
     image_info.push_back(std::move(cur_info));
   }
   return image_info;
 }
 
+SkBitmap PDFiumPage::GetImageForOcr(int page_object_index) {
+  SkBitmap bitmap;
+
+  FPDF_PAGE page = GetPage();
+  FPDF_PAGEOBJECT page_object = FPDFPage_GetObject(page, page_object_index);
+
+  if (FPDFPageObj_GetType(page_object) != FPDF_PAGEOBJ_IMAGE) {
+    return bitmap;
+  }
+
+  // OCR needs the image with the highest available quality. To get it, the
+  // image transform matrix is reset to no-scale, the bitmap is extracted,
+  // and then the original matrix is restored.
+  FS_MATRIX original_matrix;
+  if (!FPDFPageObj_GetMatrix(page_object, &original_matrix)) {
+    return bitmap;
+  }
+
+  // Get the actual image size.
+  unsigned int width;
+  unsigned int height;
+  if (!FPDFImageObj_GetImagePixelSize(page_object, &width, &height)) {
+    return bitmap;
+  }
+
+  // Resize the matrix to actual size.
+  FS_MATRIX new_matrix = {static_cast<float>(width),  0, 0,
+                          static_cast<float>(height), 0, 0};
+  if (!FPDFPageObj_SetMatrix(page_object, &new_matrix)) {
+    return bitmap;
+  }
+
+  ScopedFPDFBitmap raw_bitmap(
+      FPDFImageObj_GetRenderedBitmap(engine_->doc(), page, page_object));
+
+  // Restore the original matrix.
+  CHECK(FPDFPageObj_SetMatrix(page_object, &original_matrix));
+
+  if (!raw_bitmap) {
+    return SkBitmap();
+  }
+
+  CHECK_EQ(FPDFBitmap_GetFormat(raw_bitmap.get()), FPDFBitmap_BGRA);
+  SkImageInfo info =
+      SkImageInfo::Make(FPDFBitmap_GetWidth(raw_bitmap.get()),
+                        FPDFBitmap_GetHeight(raw_bitmap.get()),
+                        kBGRA_8888_SkColorType, kOpaque_SkAlphaType);
+  const size_t row_bytes = FPDFBitmap_GetStride(raw_bitmap.get());
+  SkPixmap pixels(info, FPDFBitmap_GetBuffer(raw_bitmap.get()), row_bytes);
+  if (!bitmap.tryAllocPixels(info, row_bytes)) {
+    return bitmap;
+  }
+  bitmap.writePixels(pixels);
+
+  SkBitmapOperations::RotationAmount rotation;
+  switch (FPDFPage_GetRotation(page)) {
+    case 0:
+      return bitmap;
+    case 1:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_90_CW;
+      break;
+    case 2:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_180_CW;
+      break;
+    case 3:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_270_CW;
+      break;
+  }
+
+  return SkBitmapOperations::Rotate(bitmap, rotation);
+}
+
 std::vector<AccessibilityHighlightInfo> PDFiumPage::GetHighlightInfo(
     const std::vector<AccessibilityTextRunInfo>& text_runs) {
   std::vector<AccessibilityHighlightInfo> highlight_info;
@@ -1271,80 +1343,6 @@ void PDFiumPage::CalculateImages() {
 
   if (!marked_content_id_image_map.empty())
     PopulateImageAltText(marked_content_id_image_map);
-
-  if (!features::IsPdfOcrEnabled())
-    return;
-
-  // If requested by the user, we store the raw image data so that the OCR
-  // service can try and retrieve textual and layout information from the image.
-  // This is because alt text might be empty, or the PDF might simply be
-  // untagged for accessibility.
-  for (Image& image : images_) {
-    if (!image.alt_text.empty())
-      continue;
-
-    FPDF_PAGEOBJECT page_object =
-        FPDFPage_GetObject(page, image.page_object_index);
-
-    // OCR needs the image with the highest available quality. To get it, the
-    // image transform matrix is reset to no-scale, the bitmap is extracted,
-    // and then the original matrix is restored.
-    FS_MATRIX original_matrix;
-    if (!FPDFPageObj_GetMatrix(page_object, &original_matrix)) {
-      continue;
-    }
-
-    // Get the actual image size.
-    unsigned int width;
-    unsigned int height;
-    if (!FPDFImageObj_GetImagePixelSize(page_object, &width, &height)) {
-      continue;
-    }
-
-    // Resize the matrix to actual size.
-    FS_MATRIX new_matrix = {static_cast<float>(width),  0, 0,
-                            static_cast<float>(height), 0, 0};
-    if (!FPDFPageObj_SetMatrix(page_object, &new_matrix)) {
-      continue;
-    }
-
-    ScopedFPDFBitmap bitmap(
-        FPDFImageObj_GetRenderedBitmap(engine_->doc(), page, page_object));
-
-    // Restore the original matrix.
-    CHECK(FPDFPageObj_SetMatrix(page_object, &original_matrix));
-
-    if (!bitmap)
-      continue;
-
-    CHECK_EQ(FPDFBitmap_GetFormat(bitmap.get()), FPDFBitmap_BGRA);
-    SkImageInfo info = SkImageInfo::Make(
-        FPDFBitmap_GetWidth(bitmap.get()), FPDFBitmap_GetHeight(bitmap.get()),
-        kBGRA_8888_SkColorType, kOpaque_SkAlphaType);
-    const size_t row_bytes = FPDFBitmap_GetStride(bitmap.get());
-    SkPixmap pixels(info, FPDFBitmap_GetBuffer(bitmap.get()), row_bytes);
-    if (!image.image_data.tryAllocPixels(info, row_bytes)) {
-      continue;
-    }
-    image.image_data.writePixels(pixels);
-
-    SkBitmapOperations::RotationAmount rotation;
-    switch (FPDFPage_GetRotation(page)) {
-      case 0:
-        continue;
-      case 1:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_90_CW;
-        break;
-      case 2:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_180_CW;
-        break;
-      case 3:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_270_CW;
-        break;
-    }
-
-    image.image_data = SkBitmapOperations::Rotate(image.image_data, rotation);
-  }
 }
 
 void PDFiumPage::PopulateImageAltText(

pdf/pdfium/pdfium_page.h

@@ -89,6 +89,9 @@ class PDFiumPage {
   // `image_data` field.
   std::vector<AccessibilityImageInfo> GetImageInfo(uint32_t text_run_count);
 
+  // Returns the image as a 32-bit bitmap format for OCR.
+  SkBitmap GetImageForOcr(int page_object_index);
+
   // For all the highlights on the page, get their underlying text ranges and
   // bounding boxes.
   std::vector<AccessibilityHighlightInfo> GetHighlightInfo(
@@ -231,6 +234,7 @@ class PDFiumPage {
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, CalculateImages);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, ImageAltText);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, ImageData);
+  FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, ImageDataForNonImage);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, RotatedPageImageData);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, AnnotLinkGeneration);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, GetLinkTarget);
@@ -269,15 +273,12 @@ class PDFiumPage {
     Image(const Image& other);
     ~Image();
 
+    // Index of the object in its page.
     int page_object_index;
+
     // Alt text is available only for PDFs that are tagged for accessibility.
     std::string alt_text;
     gfx::Rect bounding_rect;
-    // Image data is only stored if the user has requested that the OCR service
-    // try to retrieve textual and layout information from this image. The
-    // bitmap will have the same size as the image in the PDF file, and will
-    // not be scaled.
-    SkBitmap image_data;
   };
 
   // Represents a highlight within the page.

pdf/pdfium/pdfium_page_unittest.cc

@@ -541,16 +541,20 @@ TEST_P(PDFiumPageImageDataTest, ImageData) {
   ASSERT_EQ(3u, page.images_.size());
 
   ASSERT_FALSE(page.images_[0].alt_text.empty());
-  EXPECT_TRUE(page.images_[0].image_data.drawsNothing());
-  EXPECT_EQ(page.images_[0].image_data.width(), 0);
-  EXPECT_EQ(page.images_[0].image_data.height(), 0);
-
-  ASSERT_TRUE(page.images_[2].alt_text.empty());
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, page.images_[0].page_object_index);
+  EXPECT_FALSE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 50);
+  EXPECT_EQ(image_bitmap.height(), 50);
 
+  ASSERT_TRUE(page.images_[1].alt_text.empty());
+  image_bitmap = engine->GetImageForOcr(/*page_index=*/0,
+                                        page.images_[1].page_object_index);
+  EXPECT_FALSE(image_bitmap.drawsNothing());
   // While the scaled image size is 20x20, `image_data` has the same size as
   // the image in the PDF file, which is 50x50, and is not scaled.
-  EXPECT_EQ(page.images_[1].image_data.width(), 50);
-  EXPECT_EQ(page.images_[1].image_data.height(), 50);
+  EXPECT_EQ(image_bitmap.width(), 50);
+  EXPECT_EQ(image_bitmap.height(), 50);
 }
 
 TEST_P(PDFiumPageImageDataTest, RotatedPageImageData) {
@@ -566,8 +570,37 @@ TEST_P(PDFiumPageImageDataTest, RotatedPageImageData) {
 
   // This page is rotated, therefore the extracted image size is 25x100 while
   // the stored image is 100x25.
-  EXPECT_EQ(page.images_[0].image_data.width(), 25);
-  EXPECT_EQ(page.images_[0].image_data.height(), 100);
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, page.images_[0].page_object_index);
+  EXPECT_EQ(image_bitmap.width(), 25);
+  EXPECT_EQ(image_bitmap.height(), 100);
+}
+
+TEST_P(PDFiumPageImageDataTest, ImageDataForNonImage) {
+  TestClient client;
+  std::unique_ptr<PDFiumEngine> engine =
+      InitializeEngine(&client, FILE_PATH_LITERAL("text_with_image.pdf"));
+  ASSERT_TRUE(engine);
+  ASSERT_EQ(1, engine->GetNumberOfPages());
+
+  PDFiumPage& page = GetPDFiumPageForTest(*engine, 0);
+  page.CalculateImages();
+  ASSERT_EQ(3u, page.images_.size());
+  ASSERT_EQ(1, page.images_[0].page_object_index);
+
+  // Existing non-image object.
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, /*image_index=*/0);
+  EXPECT_TRUE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 0);
+  EXPECT_EQ(image_bitmap.height(), 0);
+
+  // Out of range.
+  image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, /*image_index=*/1000);
+  EXPECT_TRUE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 0);
+  EXPECT_EQ(image_bitmap.height(), 0);
 }
 
 INSTANTIATE_TEST_SUITE_P(All, PDFiumPageImageDataTest, testing::Bool());