Extract high quality images from PDF only when they are needed for OCR.

32-bit bitmaps with highest available quality are extracted from PDF
files when they are loaded, so that they would be sent later to OCR
service. To avoid the memory overhead of this process, this CL
postpones image extraction to the time they are sent to OCR service,
and destroys the extracted images immediately after that.

AX-Relnotes: n/a
Bug: 1471392
Change-Id: Id337edf693d8d4a4ddd1a56d814a0d1f0e1ac5e4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4754282
Auto-Submit: Ramin Halavati <rhalavati@chromium.org>
Reviewed-by: Kyungjun Lee <kyungjunlee@google.com>
Commit-Queue: Ramin Halavati <rhalavati@chromium.org>
Commit-Queue: Lei Zhang <thestig@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1182463}

components/pdf/renderer/DEPS

@@ -12,6 +12,7 @@ include_rules = [
   "+pdf/mojom/pdf.mojom.h",
   "+pdf/pdf_accessibility_action_handler.h",
   "+pdf/pdf_accessibility_data_handler.h",
+  "+pdf/pdf_accessibility_image_fetcher.h",
   "+pdf/pdf_features.h",
   "+pdf/pdf_view_web_plugin.h",
   "+printing/buildflags/buildflags.h",

components/pdf/renderer/pdf_accessibility_tree.cc

@@ -25,6 +25,7 @@
 #include "content/public/renderer/render_frame.h"
 #include "content/public/renderer/render_thread.h"
 #include "pdf/pdf_accessibility_action_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_features.h"
 #include "third_party/blink/public/strings/grit/blink_accessibility_strings.h"
 #include "ui/accessibility/ax_enums.mojom.h"
@@ -57,10 +58,12 @@ using PdfOcrRequest = PdfAccessibilityTree::PdfOcrRequest;
 
 PdfOcrRequest::PdfOcrRequest(const ui::AXNodeID& image_node_id,
                              const chrome_pdf::AccessibilityImageInfo& image,
-                             const ui::AXNodeID& parent_node_id)
+                             const ui::AXNodeID& parent_node_id,
+                             uint32_t page_index)
     : image_node_id(image_node_id),
       image(image),
-      parent_node_id(parent_node_id) {}
+      parent_node_id(parent_node_id),
+      page_index(page_index) {}
 
 //
 // PdfOcrService
@@ -68,10 +71,13 @@ PdfOcrRequest::PdfOcrRequest(const ui::AXNodeID& image_node_id,
 
 using PdfOcrService = PdfAccessibilityTree::PdfOcrService;
 
-PdfOcrService::PdfOcrService(content::RenderFrame& render_frame,
-                             uint32_t page_count,
-                             OnOcrDataReceivedCallback callback)
-    : remaining_page_count_(page_count),
+PdfOcrService::PdfOcrService(
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher,
+    content::RenderFrame& render_frame,
+    uint32_t page_count,
+    OnOcrDataReceivedCallback callback)
+    : image_fetcher_(image_fetcher),
+      remaining_page_count_(page_count),
       on_ocr_data_received_callback_(std::move(callback)) {
   CHECK(features::IsPdfOcrEnabled());
   render_frame.GetBrowserInterfaceBroker()->GetInterface(
@@ -124,12 +130,21 @@ void PdfOcrService::OcrNextImage() {
   if (all_requests_.empty()) {
     return;
   }
-  const PdfOcrRequest request = all_requests_.front();
+  PdfOcrRequest request = all_requests_.front();
   all_requests_.pop();
+
+  SkBitmap bitmap = image_fetcher_->GetImageForOcr(
+      request.page_index, request.image.page_object_index);
+  request.image_pixel_size = gfx::SizeF(bitmap.width(), bitmap.height());
+  if (bitmap.drawsNothing()) {
+    ReceiveOcrResultsForImage(std::move(request), ui::AXTreeUpdate());
+    return;
+  }
+
   screen_ai_annotator_->PerformOcrAndReturnAXTreeUpdate(
-      request.image.image_data,
+      std::move(bitmap),
       base::BindOnce(&PdfOcrService::ReceiveOcrResultsForImage,
-                     weak_ptr_factory_.GetWeakPtr(), request));
+                     weak_ptr_factory_.GetWeakPtr(), std::move(request)));
 
   base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
                                 PdfOcrRequestStatus::kRequested);
@@ -506,14 +521,11 @@ std::unique_ptr<ui::AXNodeData> CreateStatusNodeWrapper(
   return node_wrapper;
 }
 
-gfx::Transform MakeTransformForImage(
-    const chrome_pdf::AccessibilityImageInfo& image) {
+gfx::Transform MakeTransformForImage(const gfx::RectF image_screen_size,
+                                     const gfx::SizeF image_pixel_size) {
   // Nodes created with OCR results from the image will be misaligned on screen
   // if `image_screen_size` is different from `image_pixel_size`. To address
   // this misalignment issue, an additional transform needs to be created.
-  const gfx::RectF& image_screen_size = image.bounds;
-  const gfx::RectF image_pixel_size =
-      gfx::RectF(image.image_data.width(), image.image_data.height());
   CHECK(!image_pixel_size.IsEmpty());
 
   gfx::Transform transform;
@@ -1331,9 +1343,9 @@ class PdfAccessibilityTreeBuilder {
       ui::AXNodeData* image_node = CreateImageNode(images_[i]);
       para_node->child_ids.push_back(image_node->id);
 #if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
-      if (!has_accessible_text_ && ocr_available &&
-          !images_[i].image_data.drawsNothing()) {
-        ocr_requests.emplace(image_node->id, images_[i], para_node->id);
+      if (!has_accessible_text_ && ocr_available) {
+        ocr_requests.emplace(image_node->id, images_[i], para_node->id,
+                             page_index_);
       }
 #endif
     }
@@ -1409,12 +1421,15 @@ class PdfAccessibilityTreeBuilder {
 
 PdfAccessibilityTree::PdfAccessibilityTree(
     content::RenderFrame* render_frame,
-    chrome_pdf::PdfAccessibilityActionHandler* action_handler)
+    chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher)
     : content::RenderFrameObserver(render_frame),
       render_frame_(render_frame),
-      action_handler_(action_handler) {
+      action_handler_(action_handler),
+      image_fetcher_(image_fetcher) {
   DCHECK(render_frame);
   DCHECK(action_handler_);
+  DCHECK(image_fetcher_);
   MaybeHandleAccessibilityChange(/*always_load_or_reload_accessibility=*/false);
 }
 
@@ -2107,14 +2122,17 @@ void PdfAccessibilityTree::OnOcrDataReceived(
     // would be more convenient and less complex if an `ui::AXTree` was never
     // constructed and if the `ui::AXTreeSource` was able to use the collection
     // of `nodes_` directly.
-    base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
-                                  PdfOcrRequestStatus::kPerformed);
 
     if (tree_update.nodes.empty()) {
       VLOG(1) << "Empty OCR data received.";
+      // TODO(crbug.com/1471392): Create an empty update and continue. This can
+      // happen if OCR returns an empty result, or the image draws nothing.
       return;
     }
 
+    base::UmaHistogramEnumeration("Accessibility.PdfOcr.PDFImages",
+                                  PdfOcrRequestStatus::kPerformed);
+
     // Update the flag if OCR extracted text from any images. This flag will be
     // used to update the status node to notify users of it.
     was_text_converted_from_image_ = true;
@@ -2145,7 +2163,8 @@ void PdfAccessibilityTree::OnOcrDataReceived(
     // transform, nodes created from OCR results will have misaligned bounding
     // boxes. This transform will be applied to all nodes from OCR results
     // below.
-    gfx::Transform transform = MakeTransformForImage(ocr_request.image);
+    gfx::Transform transform = MakeTransformForImage(
+        ocr_request.image.bounds, ocr_request.image_pixel_size);
 
     // Count each detected language and find out the most detected language in
     // OCR result. Then record the most detected language in UMA.
@@ -2231,7 +2250,7 @@ void PdfAccessibilityTree::OnOcrDataReceived(
 void PdfAccessibilityTree::CreateOcrService() {
   VLOG(2) << "Creating OCR service.";
   ocr_service_ = std::make_unique<PdfOcrService>(
-      *render_frame_, page_count_,
+      image_fetcher_, *render_frame_, page_count_,
       base::BindRepeating(&PdfAccessibilityTree::OnOcrDataReceived,
                           weak_ptr_factory_.GetWeakPtr()));
 }

components/pdf/renderer/pdf_accessibility_tree.h

@@ -35,6 +35,7 @@
 namespace chrome_pdf {
 
 class PdfAccessibilityActionHandler;
+class PdfAccessibilityImageFetcher;
 
 }  // namespace chrome_pdf
 
@@ -72,14 +73,19 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
   struct PdfOcrRequest {
     PdfOcrRequest(const ui::AXNodeID& image_node_id,
                   const chrome_pdf::AccessibilityImageInfo& image,
-                  const ui::AXNodeID& parent_node_id);
+                  const ui::AXNodeID& parent_node_id,
+                  uint32_t page_index);
 
     const ui::AXNodeID image_node_id;
     const chrome_pdf::AccessibilityImageInfo image;
     const ui::AXNodeID parent_node_id;
+    const uint32_t page_index;
     // This boolean indicates which request corresponds to the last image on
     // each page.
     bool is_last_on_page = false;
+
+    // This field is set after the image is extracted from PDF.
+    gfx::SizeF image_pixel_size;
   };
 
   // Manages the connection to the OCR Service via Mojo, and ensures that
@@ -90,7 +96,8 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
         std::vector<PdfOcrRequest> ocr_requests,
         std::vector<ui::AXTreeUpdate> tree_updates)>;
 
-    PdfOcrService(content::RenderFrame& render_frame,
+    PdfOcrService(chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher,
+                  content::RenderFrame& render_frame,
                   uint32_t page_count,
                   OnOcrDataReceivedCallback callback);
 
@@ -124,6 +131,9 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
     void ReceiveOcrResultsForImage(PdfOcrRequest request,
                                    const ui::AXTreeUpdate& tree_update);
 
+    // `image_fetcher_` owns `this`.
+    chrome_pdf::PdfAccessibilityImageFetcher* const image_fetcher_;
+
     uint32_t remaining_page_count_;
     // True if there are pending OCR requests. Used to determine if `OcrPage`
     // should call `OcrNextImage` or if the next call to
@@ -149,7 +159,8 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
 
   PdfAccessibilityTree(
       content::RenderFrame* render_frame,
-      chrome_pdf::PdfAccessibilityActionHandler* action_handler);
+      chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+      chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher);
   ~PdfAccessibilityTree() override;
 
   static bool IsDataFromPluginValid(
@@ -305,6 +316,7 @@ class PdfAccessibilityTree : public content::PluginAXTreeSource,
 
   // Unowned. Must outlive `this`.
   chrome_pdf::PdfAccessibilityActionHandler* const action_handler_;
+  chrome_pdf::PdfAccessibilityImageFetcher* const image_fetcher_;
 
   // `zoom_` signifies the zoom level set in for the browser content.
   // `scale_` signifies the scale level set by user. Scale is applied

components/pdf/renderer/pdf_view_web_plugin_client.cc

@@ -255,8 +255,10 @@ void PdfViewWebPluginClient::RecordComputedAction(const std::string& action) {
 
 std::unique_ptr<chrome_pdf::PdfAccessibilityDataHandler>
 PdfViewWebPluginClient::CreateAccessibilityDataHandler(
-    chrome_pdf::PdfAccessibilityActionHandler* action_handler) {
-  return std::make_unique<PdfAccessibilityTree>(render_frame_, action_handler);
+    chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+    chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher) {
+  return std::make_unique<PdfAccessibilityTree>(render_frame_, action_handler,
+                                                image_fetcher);
 }
 
 }  // namespace pdf

components/pdf/renderer/pdf_view_web_plugin_client.h

@@ -78,7 +78,8 @@ class PdfViewWebPluginClient : public chrome_pdf::PdfViewWebPlugin::Client {
   void RecordComputedAction(const std::string& action) override;
   std::unique_ptr<chrome_pdf::PdfAccessibilityDataHandler>
   CreateAccessibilityDataHandler(
-      chrome_pdf::PdfAccessibilityActionHandler* action_handler) override;
+      chrome_pdf::PdfAccessibilityActionHandler* action_handler,
+      chrome_pdf::PdfAccessibilityImageFetcher* image_fetcher) override;
 
  private:
   blink::WebLocalFrame* GetFrame() const;

pdf/BUILD.gn

@@ -217,6 +217,7 @@ if (enable_pdf) {
       "accessibility_structs.h",
       "pdf_accessibility_action_handler.h",
       "pdf_accessibility_data_handler.h",
+      "pdf_accessibility_image_fetcher.h",
     ]
 
     configs += [ ":strict" ]

pdf/accessibility_structs.cc

@@ -77,11 +77,11 @@ AccessibilityImageInfo::AccessibilityImageInfo() = default;
 AccessibilityImageInfo::AccessibilityImageInfo(const std::string& alt_text,
                                                uint32_t text_run_index,
                                                const gfx::RectF& bounds,
-                                               const SkBitmap& image_data)
+                                               int32_t page_object_index)
     : alt_text(alt_text),
       text_run_index(text_run_index),
       bounds(bounds),
-      image_data(image_data) {}
+      page_object_index(page_object_index) {}
 
 AccessibilityImageInfo::AccessibilityImageInfo(
     const AccessibilityImageInfo& other) = default;

pdf/accessibility_structs.h

@@ -132,7 +132,7 @@ struct AccessibilityImageInfo {
   AccessibilityImageInfo(const std::string& alt_text,
                          uint32_t text_run_index,
                          const gfx::RectF& bounds,
-                         const SkBitmap& image_data);
+                         int32_t page_object_index);
   AccessibilityImageInfo(const AccessibilityImageInfo& other);
   ~AccessibilityImageInfo();
 
@@ -147,9 +147,8 @@ struct AccessibilityImageInfo {
   // Bounding box of the image.
   gfx::RectF bounds;
 
-  // Only populated if `alt_text` is empty or unavailable, and if the user has
-  // requested that the OCR service tag the PDF so that it is made accessible.
-  SkBitmap image_data;
+  // Index of the image object in its page.
+  int32_t page_object_index;
 };
 
 struct AccessibilityHighlightInfo {

pdf/pdf_accessibility_image_fetcher.h

@@ -0,0 +1,22 @@
+// Copyright 2023 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_
+#define PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_
+
+class SkBitmap;
+
+namespace chrome_pdf {
+
+class PdfAccessibilityImageFetcher {
+ public:
+  virtual ~PdfAccessibilityImageFetcher() = default;
+  // Fetches the image as a 32-bit bitmap for OCR.
+  virtual SkBitmap GetImageForOcr(int32_t page_index,
+                                  int32_t page_object_index) = 0;
+};
+
+}  // namespace chrome_pdf
+
+#endif  // PDF_PDF_ACCESSIBILITY_IMAGE_FETCHER_H_

pdf/pdf_engine.h

@@ -408,6 +408,8 @@ class PDFEngine {
   virtual std::vector<AccessibilityImageInfo> GetImageInfo(
       int page_index,
       uint32_t text_run_count) = 0;
+  // Returns the image as a 32-bit bitmap format for OCR.
+  virtual SkBitmap GetImageForOcr(int page_index, int image_index) = 0;
   // For all the highlights in page `page_index`, get their underlying text
   // ranges and bounding boxes.
   virtual std::vector<AccessibilityHighlightInfo> GetHighlightInfo(

pdf/pdf_view_web_plugin.cc

@@ -274,7 +274,8 @@ std::unique_ptr<PDFiumEngine> PdfViewWebPlugin::Client::CreateEngine(
 
 std::unique_ptr<PdfAccessibilityDataHandler>
 PdfViewWebPlugin::Client::CreateAccessibilityDataHandler(
-    PdfAccessibilityActionHandler* action_handler) {
+    PdfAccessibilityActionHandler* action_handler,
+    PdfAccessibilityImageFetcher* image_fetcher) {
   return nullptr;
 }
 
@@ -286,7 +287,7 @@ PdfViewWebPlugin::PdfViewWebPlugin(
       pdf_service_(std::move(pdf_service)),
       initial_params_(params),
       pdf_accessibility_data_handler_(
-          client_->CreateAccessibilityDataHandler(this)) {
+          client_->CreateAccessibilityDataHandler(this, this)) {
   DCHECK(pdf_service_);
   pdf_service_->SetListener(listener_receiver_.BindNewPipeAndPassRemote());
 }
@@ -1929,6 +1930,11 @@ void PdfViewWebPlugin::EnableAccessibility() {
   LoadOrReloadAccessibility();
 }
 
+SkBitmap PdfViewWebPlugin::GetImageForOcr(int32_t page_index,
+                                          int32_t page_object_index) {
+  return engine_->GetImageForOcr(page_index, page_object_index);
+}
+
 void PdfViewWebPlugin::HandleAccessibilityAction(
     const AccessibilityActionData& action_data) {
   engine_->HandleAccessibilityAction(action_data);

pdf/pdf_view_web_plugin.h

@@ -27,6 +27,7 @@
 #include "pdf/mojom/pdf.mojom.h"
 #include "pdf/paint_manager.h"
 #include "pdf/pdf_accessibility_action_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_engine.h"
 #include "pdf/pdfium/pdfium_form_filler.h"
 #include "pdf/post_message_receiver.h"
@@ -82,6 +83,7 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
                                public PostMessageReceiver::Client,
                                public PaintManager::Client,
                                public PdfAccessibilityActionHandler,
+                               public PdfAccessibilityImageFetcher,
                                public PreviewModeClient::Client {
  public:
   // Do not save files larger than 100 MB. This cap should be kept in sync with
@@ -219,7 +221,8 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
     // client.
     virtual std::unique_ptr<PdfAccessibilityDataHandler>
     CreateAccessibilityDataHandler(
-        PdfAccessibilityActionHandler* action_handler);
+        PdfAccessibilityActionHandler* action_handler,
+        PdfAccessibilityImageFetcher* image_fetcher);
   };
 
   PdfViewWebPlugin(std::unique_ptr<Client> client,
@@ -381,6 +384,10 @@ class PdfViewWebPlugin final : public PDFEngine::Client,
       const AccessibilityActionData& action_data) override;
   void LoadOrReloadAccessibility() override;
 
+  // PdfAccessibilityImageFetcher:
+  SkBitmap GetImageForOcr(int32_t page_index,
+                          int32_t page_object_index) override;
+
   // PreviewModeClient::Client:
   void PreviewDocumentLoadComplete() override;
   void PreviewDocumentLoadFailed() override;

pdf/pdf_view_web_plugin_unittest.cc

@@ -37,6 +37,7 @@
 #include "pdf/mojom/pdf.mojom.h"
 #include "pdf/paint_ready_rect.h"
 #include "pdf/pdf_accessibility_data_handler.h"
+#include "pdf/pdf_accessibility_image_fetcher.h"
 #include "pdf/pdf_features.h"
 #include "pdf/test/mock_web_associated_url_loader.h"
 #include "pdf/test/test_helpers.h"
@@ -308,7 +309,7 @@ class FakePdfViewWebPluginClient : public PdfViewWebPlugin::Client {
 
   MOCK_METHOD(std::unique_ptr<PdfAccessibilityDataHandler>,
               CreateAccessibilityDataHandler,
-              (PdfAccessibilityActionHandler*),
+              (PdfAccessibilityActionHandler*, PdfAccessibilityImageFetcher*),
               (override));
 };
 

pdf/pdfium/pdfium_engine.cc

@@ -2618,6 +2618,11 @@ std::vector<AccessibilityImageInfo> PDFiumEngine::GetImageInfo(
   return pages_[page_index]->GetImageInfo(text_run_count);
 }
 
+SkBitmap PDFiumEngine::GetImageForOcr(int page_index, int image_index) {
+  DCHECK(PageIndexInBounds(page_index));
+  return pages_[page_index]->GetImageForOcr(image_index);
+}
+
 std::vector<AccessibilityHighlightInfo> PDFiumEngine::GetHighlightInfo(
     int page_index,
     const std::vector<AccessibilityTextRunInfo>& text_runs) {

pdf/pdfium/pdfium_engine.h

@@ -158,6 +158,7 @@ class PDFiumEngine : public PDFEngine,
   std::vector<AccessibilityImageInfo> GetImageInfo(
       int page_index,
       uint32_t text_run_count) override;
+  SkBitmap GetImageForOcr(int page_index, int image_index) override;
   std::vector<AccessibilityHighlightInfo> GetHighlightInfo(
       int page_index,
       const std::vector<AccessibilityTextRunInfo>& text_runs) override;

pdf/pdfium/pdfium_page.cc

@@ -770,12 +770,84 @@ std::vector<AccessibilityImageInfo> PDFiumPage::GetImageInfo(
     cur_info.bounds =
         gfx::RectF(image.bounding_rect.x(), image.bounding_rect.y(),
                    image.bounding_rect.width(), image.bounding_rect.height());
-    cur_info.image_data = image.image_data;
+    cur_info.page_object_index = image.page_object_index;
     image_info.push_back(std::move(cur_info));
   }
   return image_info;
 }
 
+SkBitmap PDFiumPage::GetImageForOcr(int page_object_index) {
+  SkBitmap bitmap;
+
+  FPDF_PAGE page = GetPage();
+  FPDF_PAGEOBJECT page_object = FPDFPage_GetObject(page, page_object_index);
+
+  if (FPDFPageObj_GetType(page_object) != FPDF_PAGEOBJ_IMAGE) {
+    return bitmap;
+  }
+
+  // OCR needs the image with the highest available quality. To get it, the
+  // image transform matrix is reset to no-scale, the bitmap is extracted,
+  // and then the original matrix is restored.
+  FS_MATRIX original_matrix;
+  if (!FPDFPageObj_GetMatrix(page_object, &original_matrix)) {
+    return bitmap;
+  }
+
+  // Get the actual image size.
+  unsigned int width;
+  unsigned int height;
+  if (!FPDFImageObj_GetImagePixelSize(page_object, &width, &height)) {
+    return bitmap;
+  }
+
+  // Resize the matrix to actual size.
+  FS_MATRIX new_matrix = {static_cast<float>(width),  0, 0,
+                          static_cast<float>(height), 0, 0};
+  if (!FPDFPageObj_SetMatrix(page_object, &new_matrix)) {
+    return bitmap;
+  }
+
+  ScopedFPDFBitmap raw_bitmap(
+      FPDFImageObj_GetRenderedBitmap(engine_->doc(), page, page_object));
+
+  // Restore the original matrix.
+  CHECK(FPDFPageObj_SetMatrix(page_object, &original_matrix));
+
+  if (!raw_bitmap) {
+    return SkBitmap();
+  }
+
+  CHECK_EQ(FPDFBitmap_GetFormat(raw_bitmap.get()), FPDFBitmap_BGRA);
+  SkImageInfo info =
+      SkImageInfo::Make(FPDFBitmap_GetWidth(raw_bitmap.get()),
+                        FPDFBitmap_GetHeight(raw_bitmap.get()),
+                        kBGRA_8888_SkColorType, kOpaque_SkAlphaType);
+  const size_t row_bytes = FPDFBitmap_GetStride(raw_bitmap.get());
+  SkPixmap pixels(info, FPDFBitmap_GetBuffer(raw_bitmap.get()), row_bytes);
+  if (!bitmap.tryAllocPixels(info, row_bytes)) {
+    return bitmap;
+  }
+  bitmap.writePixels(pixels);
+
+  SkBitmapOperations::RotationAmount rotation;
+  switch (FPDFPage_GetRotation(page)) {
+    case 0:
+      return bitmap;
+    case 1:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_90_CW;
+      break;
+    case 2:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_180_CW;
+      break;
+    case 3:
+      rotation = SkBitmapOperations::RotationAmount::ROTATION_270_CW;
+      break;
+  }
+
+  return SkBitmapOperations::Rotate(bitmap, rotation);
+}
+
 std::vector<AccessibilityHighlightInfo> PDFiumPage::GetHighlightInfo(
     const std::vector<AccessibilityTextRunInfo>& text_runs) {
   std::vector<AccessibilityHighlightInfo> highlight_info;
@@ -1271,80 +1343,6 @@ void PDFiumPage::CalculateImages() {
 
   if (!marked_content_id_image_map.empty())
     PopulateImageAltText(marked_content_id_image_map);
-
-  if (!features::IsPdfOcrEnabled())
-    return;
-
-  // If requested by the user, we store the raw image data so that the OCR
-  // service can try and retrieve textual and layout information from the image.
-  // This is because alt text might be empty, or the PDF might simply be
-  // untagged for accessibility.
-  for (Image& image : images_) {
-    if (!image.alt_text.empty())
-      continue;
-
-    FPDF_PAGEOBJECT page_object =
-        FPDFPage_GetObject(page, image.page_object_index);
-
-    // OCR needs the image with the highest available quality. To get it, the
-    // image transform matrix is reset to no-scale, the bitmap is extracted,
-    // and then the original matrix is restored.
-    FS_MATRIX original_matrix;
-    if (!FPDFPageObj_GetMatrix(page_object, &original_matrix)) {
-      continue;
-    }
-
-    // Get the actual image size.
-    unsigned int width;
-    unsigned int height;
-    if (!FPDFImageObj_GetImagePixelSize(page_object, &width, &height)) {
-      continue;
-    }
-
-    // Resize the matrix to actual size.
-    FS_MATRIX new_matrix = {static_cast<float>(width),  0, 0,
-                            static_cast<float>(height), 0, 0};
-    if (!FPDFPageObj_SetMatrix(page_object, &new_matrix)) {
-      continue;
-    }
-
-    ScopedFPDFBitmap bitmap(
-        FPDFImageObj_GetRenderedBitmap(engine_->doc(), page, page_object));
-
-    // Restore the original matrix.
-    CHECK(FPDFPageObj_SetMatrix(page_object, &original_matrix));
-
-    if (!bitmap)
-      continue;
-
-    CHECK_EQ(FPDFBitmap_GetFormat(bitmap.get()), FPDFBitmap_BGRA);
-    SkImageInfo info = SkImageInfo::Make(
-        FPDFBitmap_GetWidth(bitmap.get()), FPDFBitmap_GetHeight(bitmap.get()),
-        kBGRA_8888_SkColorType, kOpaque_SkAlphaType);
-    const size_t row_bytes = FPDFBitmap_GetStride(bitmap.get());
-    SkPixmap pixels(info, FPDFBitmap_GetBuffer(bitmap.get()), row_bytes);
-    if (!image.image_data.tryAllocPixels(info, row_bytes)) {
-      continue;
-    }
-    image.image_data.writePixels(pixels);
-
-    SkBitmapOperations::RotationAmount rotation;
-    switch (FPDFPage_GetRotation(page)) {
-      case 0:
-        continue;
-      case 1:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_90_CW;
-        break;
-      case 2:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_180_CW;
-        break;
-      case 3:
-        rotation = SkBitmapOperations::RotationAmount::ROTATION_270_CW;
-        break;
-    }
-
-    image.image_data = SkBitmapOperations::Rotate(image.image_data, rotation);
-  }
 }
 
 void PDFiumPage::PopulateImageAltText(

pdf/pdfium/pdfium_page.h

@@ -89,6 +89,9 @@ class PDFiumPage {
   // `image_data` field.
   std::vector<AccessibilityImageInfo> GetImageInfo(uint32_t text_run_count);
 
+  // Returns the image as a 32-bit bitmap format for OCR.
+  SkBitmap GetImageForOcr(int page_object_index);
+
   // For all the highlights on the page, get their underlying text ranges and
   // bounding boxes.
   std::vector<AccessibilityHighlightInfo> GetHighlightInfo(
@@ -231,6 +234,7 @@ class PDFiumPage {
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, CalculateImages);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, ImageAltText);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, ImageData);
+  FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, ImageDataForNonImage);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, RotatedPageImageData);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, AnnotLinkGeneration);
   FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, GetLinkTarget);
@@ -269,15 +273,12 @@ class PDFiumPage {
     Image(const Image& other);
     ~Image();
 
+    // Index of the object in its page.
     int page_object_index;
+
     // Alt text is available only for PDFs that are tagged for accessibility.
     std::string alt_text;
     gfx::Rect bounding_rect;
-    // Image data is only stored if the user has requested that the OCR service
-    // try to retrieve textual and layout information from this image. The
-    // bitmap will have the same size as the image in the PDF file, and will
-    // not be scaled.
-    SkBitmap image_data;
   };
 
   // Represents a highlight within the page.

pdf/pdfium/pdfium_page_unittest.cc

@@ -541,16 +541,20 @@ TEST_P(PDFiumPageImageDataTest, ImageData) {
   ASSERT_EQ(3u, page.images_.size());
 
   ASSERT_FALSE(page.images_[0].alt_text.empty());
-  EXPECT_TRUE(page.images_[0].image_data.drawsNothing());
-  EXPECT_EQ(page.images_[0].image_data.width(), 0);
-  EXPECT_EQ(page.images_[0].image_data.height(), 0);
-
-  ASSERT_TRUE(page.images_[2].alt_text.empty());
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, page.images_[0].page_object_index);
+  EXPECT_FALSE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 50);
+  EXPECT_EQ(image_bitmap.height(), 50);
 
+  ASSERT_TRUE(page.images_[1].alt_text.empty());
+  image_bitmap = engine->GetImageForOcr(/*page_index=*/0,
+                                        page.images_[1].page_object_index);
+  EXPECT_FALSE(image_bitmap.drawsNothing());
   // While the scaled image size is 20x20, `image_data` has the same size as
   // the image in the PDF file, which is 50x50, and is not scaled.
-  EXPECT_EQ(page.images_[1].image_data.width(), 50);
-  EXPECT_EQ(page.images_[1].image_data.height(), 50);
+  EXPECT_EQ(image_bitmap.width(), 50);
+  EXPECT_EQ(image_bitmap.height(), 50);
 }
 
 TEST_P(PDFiumPageImageDataTest, RotatedPageImageData) {
@@ -566,8 +570,37 @@ TEST_P(PDFiumPageImageDataTest, RotatedPageImageData) {
 
   // This page is rotated, therefore the extracted image size is 25x100 while
   // the stored image is 100x25.
-  EXPECT_EQ(page.images_[0].image_data.width(), 25);
-  EXPECT_EQ(page.images_[0].image_data.height(), 100);
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, page.images_[0].page_object_index);
+  EXPECT_EQ(image_bitmap.width(), 25);
+  EXPECT_EQ(image_bitmap.height(), 100);
+}
+
+TEST_P(PDFiumPageImageDataTest, ImageDataForNonImage) {
+  TestClient client;
+  std::unique_ptr<PDFiumEngine> engine =
+      InitializeEngine(&client, FILE_PATH_LITERAL("text_with_image.pdf"));
+  ASSERT_TRUE(engine);
+  ASSERT_EQ(1, engine->GetNumberOfPages());
+
+  PDFiumPage& page = GetPDFiumPageForTest(*engine, 0);
+  page.CalculateImages();
+  ASSERT_EQ(3u, page.images_.size());
+  ASSERT_EQ(1, page.images_[0].page_object_index);
+
+  // Existing non-image object.
+  SkBitmap image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, /*image_index=*/0);
+  EXPECT_TRUE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 0);
+  EXPECT_EQ(image_bitmap.height(), 0);
+
+  // Out of range.
+  image_bitmap = engine->GetImageForOcr(
+      /*page_index=*/0, /*image_index=*/1000);
+  EXPECT_TRUE(image_bitmap.drawsNothing());
+  EXPECT_EQ(image_bitmap.width(), 0);
+  EXPECT_EQ(image_bitmap.height(), 0);
 }
 
 INSTANTIATE_TEST_SUITE_P(All, PDFiumPageImageDataTest, testing::Bool());