Keep track of OCR generated words in PDFs.

Adds `is_ocr_generated` to the `AccessibilityTextRunInfo` for searchify
generated text, to be used in accessibility tree generation.

AX-Relnotes: n/a
Bug: 360803943
Change-Id: I29d44a43b70842a07b89a5c7615da4f5995e96f2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5893193
Commit-Queue: Ramin Halavati <rhalavati@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1363525}

pdf/accessibility_structs.cc

@@ -48,7 +48,23 @@ AccessibilityTextRunInfo::AccessibilityTextRunInfo(
     const gfx::RectF& bounds,
     AccessibilityTextDirection direction,
     const AccessibilityTextStyleInfo& style)
-    : len(len), bounds(bounds), direction(direction), style(style) {}
+    : AccessibilityTextRunInfo(len,
+                               bounds,
+                               direction,
+                               style,
+                               /*is_searchified=*/false) {}
+
+AccessibilityTextRunInfo::AccessibilityTextRunInfo(
+    uint32_t len,
+    const gfx::RectF& bounds,
+    AccessibilityTextDirection direction,
+    const AccessibilityTextStyleInfo& style,
+    bool is_searchified)
+    : len(len),
+      bounds(bounds),
+      direction(direction),
+      style(style),
+      is_searchified(is_searchified) {}
 
 AccessibilityTextRunInfo::AccessibilityTextRunInfo(
     const AccessibilityTextRunInfo& other) = default;

pdf/accessibility_structs.h

@@ -87,6 +87,11 @@ struct AccessibilityTextRunInfo {
                            const gfx::RectF& bounds,
                            AccessibilityTextDirection direction,
                            const AccessibilityTextStyleInfo& style);
+  AccessibilityTextRunInfo(uint32_t len,
+                           const gfx::RectF& bounds,
+                           AccessibilityTextDirection direction,
+                           const AccessibilityTextStyleInfo& style,
+                           bool is_searchified);
   AccessibilityTextRunInfo(const AccessibilityTextRunInfo& other);
   ~AccessibilityTextRunInfo();
 
@@ -94,6 +99,7 @@ struct AccessibilityTextRunInfo {
   gfx::RectF bounds;
   AccessibilityTextDirection direction = AccessibilityTextDirection::kNone;
   AccessibilityTextStyleInfo style;
+  bool is_searchified = false;
 };
 
 struct AccessibilityCharInfo {

pdf/pdfium/pdfium_on_demand_searchifier.cc

@@ -162,6 +162,7 @@ void PDFiumOnDemandSearchifier::OnGotOcrResult(
     screen_ai::mojom::VisualAnnotationPtr annotation) {
   CHECK_EQ(state_, State::kWaitingForResults);
   if (annotation) {
+    current_page_->OnSearchifyGotOcrResult();
     FPDF_PAGEOBJECT image =
         FPDFPage_GetObject(current_page_->GetPage(), image_index);
     AddTextOnImage(engine_->doc(), current_page_->GetPage(), font_.get(), image,

pdf/pdfium/pdfium_on_demand_searchifier_unittest.cc

@@ -132,9 +132,12 @@ class PDFiumOnDemandSearchifierTest : public PDFiumTestBase {
 TEST_P(PDFiumOnDemandSearchifierTest, NoImage) {
   CreateEngine(FILE_PATH_LITERAL("hello_world2.pdf"));
 
+  PDFiumPage& page = GetPDFiumPageForTest(*engine(), 0);
+
   // Load the page to trigger searchify checking.
-  engine()->GetPage(0)->GetPage();
+  page.GetPage();
   ASSERT_FALSE(engine()->PageNeedsSearchify(0));
+  EXPECT_FALSE(page.IsPageSearchified());
 
   // Searchifier should not be created as it's not needed yet.
   ASSERT_FALSE(engine()->GetSearchifierForTesting());
@@ -143,8 +146,10 @@ TEST_P(PDFiumOnDemandSearchifierTest, NoImage) {
 TEST_P(PDFiumOnDemandSearchifierTest, OnePageWithImages) {
   CreateEngine(FILE_PATH_LITERAL("image_alt_text.pdf"));
 
+  PDFiumPage& page = GetPDFiumPageForTest(*engine(), 0);
+
   // Load the page to trigger searchify checking.
-  engine()->GetPage(0)->GetPage();
+  page.GetPage();
   ASSERT_TRUE(engine()->PageNeedsSearchify(0));
 
   PDFiumOnDemandSearchifier* searchifier = engine()->GetSearchifierForTesting();
@@ -158,9 +163,10 @@ TEST_P(PDFiumOnDemandSearchifierTest, OnePageWithImages) {
   WaitUntilIdle(searchifier, future.GetCallback());
   ASSERT_TRUE(future.Wait());
   ASSERT_EQ(performed_ocrs(), 2);
+  EXPECT_TRUE(page.IsPageSearchified());
 
   // The page has two images.
-  std::string page_text = GetPageText(engine()->GetPage(0));
+  std::string page_text = GetPageText(&page);
   ASSERT_EQ(page_text, "OCR Text 0\r\nOCR Text 1");
 }
 

pdf/pdfium/pdfium_page.cc

@@ -35,7 +35,6 @@
 #include "third_party/pdfium/public/fpdf_catalog.h"
 #include "third_party/pdfium/public/fpdf_edit.h"
 #include "third_party/pdfium/public/fpdfview.h"
-#include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/accessibility/accessibility_features.h"
 #include "ui/gfx/geometry/point.h"
 #include "ui/gfx/geometry/point_f.h"
@@ -45,7 +44,11 @@
 #include "ui/gfx/geometry/vector2d.h"
 #include "ui/gfx/geometry/vector2d_f.h"
 #include "ui/gfx/range/range.h"
+
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+#include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/gfx/skbitmap_operations.h"
+#endif
 
 using printing::ConvertUnitFloat;
 using printing::kPixelsPerInch;
@@ -387,7 +390,7 @@ PDFiumPage::LinkTarget::LinkTarget(const LinkTarget& other) = default;
 PDFiumPage::LinkTarget::~LinkTarget() = default;
 
 PDFiumPage::PDFiumPage(PDFiumEngine* engine, int i)
-    : engine_(engine), index_(i), available_(false) {}
+    : engine_(engine), index_(i) {}
 
 PDFiumPage::PDFiumPage(PDFiumPage&& that) = default;
 
@@ -502,6 +505,11 @@ std::optional<AccessibilityTextRunInfo> PDFiumPage::GetTextRunInfo(
   if (start_char_index < 0 || start_char_index >= chars_count)
     return std::nullopt;
 
+  AccessibilityTextRunInfo info;
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+  info.is_searchified = IsCharacterGeneratedBySearchify(start_char_index);
+#endif
+
   int actual_start_char_index = GetFirstNonUnicodeWhiteSpaceCharIndex(
       text_page, start_char_index, chars_count);
   // Check to see if GetFirstNonUnicodeWhiteSpaceCharIndex() iterated through
@@ -510,7 +518,6 @@ std::optional<AccessibilityTextRunInfo> PDFiumPage::GetTextRunInfo(
     // If so, `info.len` needs to take the number of characters
     // iterated into account.
     DCHECK_GT(actual_start_char_index, start_char_index);
-    AccessibilityTextRunInfo info;
     info.len = chars_count - start_char_index;
     return info;
   }
@@ -527,7 +534,6 @@ std::optional<AccessibilityTextRunInfo> PDFiumPage::GetTextRunInfo(
 
   // Set text run's style info from the first character of the text run.
   FPDF_PAGEOBJECT text_object = FPDFText_GetTextObject(text_page, char_index);
-  AccessibilityTextRunInfo info;
   info.style = CalculateTextRunStyleInfo(text_object);
 
   gfx::RectF start_char_rect =
@@ -598,8 +604,9 @@ std::optional<AccessibilityTextRunInfo> PDFiumPage::GetTextRunInfo(
 
       // Heuristic: End text run if character isn't going in the same direction.
       if (char_direction !=
-          GetDirectionFromAngle(FPDFText_GetCharAngle(text_page, char_index)))
+          GetDirectionFromAngle(FPDFText_GetCharAngle(text_page, char_index))) {
         break;
+      }
 
       // Heuristic: End the text run if the difference between the text run
       // angle and the angle between the center-points of the previous and
@@ -628,8 +635,9 @@ std::optional<AccessibilityTextRunInfo> PDFiumPage::GetTextRunInfo(
           GetRotatedCharWidth(current_angle, char_rect.size()) / 2 -
           GetRotatedCharWidth(current_angle, prev_char_rect.size()) / 2;
 
-      if (distance > 2.5f * avg_char_width)
+      if (distance > 2.5f * avg_char_width) {
         break;
+      }
 
       text_run_bounds.Union(char_rect);
       prev_char_rect = char_rect;
@@ -835,6 +843,7 @@ std::vector<int> PDFiumPage::GetImageObjectIndices() {
       images_, [](const Image& image) { return image.page_object_index; });
 }
 
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
 SkBitmap PDFiumPage::GetImageForOcr(int page_object_index) {
   FPDF_PAGE page = GetPage();
   FPDF_PAGEOBJECT page_object = FPDFPage_GetObject(page, page_object_index);
@@ -859,6 +868,17 @@ SkBitmap PDFiumPage::GetImageForOcr(int page_object_index) {
   return SkBitmapOperations::Rotate(bitmap, rotation);
 }
 
+void PDFiumPage::OnSearchifyGotOcrResult() {
+  if (!IsPageSearchified()) {
+    first_searchify_generated_object_index_ = FPDFPage_CountObjects(GetPage());
+  }
+}
+
+bool PDFiumPage::IsPageSearchified() const {
+  return first_searchify_generated_object_index_ != -1;
+}
+#endif  // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
 std::vector<AccessibilityHighlightInfo> PDFiumPage::GetHighlightInfo(
     const std::vector<AccessibilityTextRunInfo>& text_runs) {
   std::vector<AccessibilityHighlightInfo> highlight_info;
@@ -1758,6 +1778,25 @@ Thumbnail PDFiumPage::GetThumbnail(float device_pixel_ratio) {
   return Thumbnail(page_size, device_pixel_ratio);
 }
 
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+bool PDFiumPage::IsCharacterGeneratedBySearchify(int char_index) {
+  if (!IsPageSearchified()) {
+    return false;
+  }
+
+  FPDF_PAGE page = GetPage();
+  int objects_count = FPDFPage_CountObjects(page);
+  FPDF_PAGEOBJECT object = FPDFText_GetTextObject(GetTextPage(), char_index);
+  for (int i = first_searchify_generated_object_index_; i < objects_count;
+       ++i) {
+    if (object == FPDFPage_GetObject(page, i)) {
+      return true;
+    }
+  }
+  return false;
+}
+#endif  // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
 void PDFiumPage::MarkAvailable() {
   available_ = true;
 

pdf/pdfium/pdfium_page.h

@@ -18,6 +18,7 @@
 #include "pdf/buildflags.h"
 #include "pdf/page_orientation.h"
 #include "pdf/ui/thumbnail.h"
+#include "services/screen_ai/buildflags/buildflags.h"
 #include "third_party/pdfium/public/cpp/fpdf_scopers.h"
 #include "third_party/pdfium/public/fpdf_doc.h"
 #include "third_party/pdfium/public/fpdf_formfill.h"
@@ -115,9 +116,18 @@ class PDFiumPage {
   // Returns the indices of image objects.
   std::vector<int> GetImageObjectIndices();
 
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
   // Returns the image as a 32-bit bitmap format for OCR.
   SkBitmap GetImageForOcr(int page_object_index);
 
+  // Called when searchify receives some results from OCR for this page.
+  // May be called several times if the page has more than one image.
+  void OnSearchifyGotOcrResult();
+
+  // Returns if searchify has run on the page.
+  bool IsPageSearchified() const;
+#endif  // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
   // For all the highlights on the page, get their underlying text ranges and
   // bounding boxes.
   std::vector<AccessibilityHighlightInfo> GetHighlightInfo(
@@ -442,6 +452,10 @@ class PDFiumPage {
   // using this page's size.
   Thumbnail GetThumbnail(float device_pixel_ratio);
 
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+  bool IsCharacterGeneratedBySearchify(int char_index);
+#endif  // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
   raw_ptr<PDFiumEngine> engine_;
   ScopedFPDFPage page_;
   ScopedFPDFTextPage text_page_;
@@ -462,7 +476,16 @@ class PDFiumPage {
   // objects.
   std::set<int> page_object_text_run_breaks_;
   base::OnceClosure thumbnail_callback_;
-  bool available_;
+  bool available_ = false;
+
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+  // The index of the first object generated by searchify. Searchify generated
+  // objects are added to the end of the page. The value is set when searchify
+  // is run on the page. If searchify does not find any alternative
+  // text for the images in the page, the value will be equal to the number of
+  // objects in the page.
+  int first_searchify_generated_object_index_ = -1;
+#endif
 };
 
 // Converts page orientations to the PDFium equivalents, as defined by