Transfers image pixels from PDFs to the renderer if OCR feature is enabled
Inaccessible PDFs will be made accessible via optical character recognition. This patch simply collects the raw pixels from every image in a PDF that is untagged, so that the OCR service can extract textual and layout information out of it. More info in: https://docs.google.com/document/d/1Xe1BBncEddGz8DYYnGdTHI0cwhJ_3BvfELCQY6T1rBw/edit?usp=sharing R=dtseng@chromium.org AX-Relnotes: n/a. Bug: 1248380 Change-Id: I938acf62339fb823b462640da8db3fc13e015905 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3516332 Reviewed-by: Ramin Halavati <rhalavati@chromium.org> Auto-Submit: Nektarios Paisios <nektar@chromium.org> Reviewed-by: Lei Zhang <thestig@chromium.org> Reviewed-by: John Abd-El-Malek <jam@chromium.org> Commit-Queue: John Abd-El-Malek <jam@chromium.org> Cr-Commit-Position: refs/heads/main@{#980985}
This commit is contained in:

committed by
Chromium LUCI CQ

parent
5fbd681585
commit
874297171d
@ -39,6 +39,7 @@ static_library("renderer") {
|
||||
"//pdf:features",
|
||||
"//pdf:pdf_view_web_plugin",
|
||||
"//printing/buildflags",
|
||||
"//skia",
|
||||
"//third_party/blink/public:blink",
|
||||
"//third_party/blink/public/strings:accessibility_strings",
|
||||
"//third_party/icu",
|
||||
|
@ -13,6 +13,7 @@ include_rules = [
|
||||
"+pdf/pdf_view_web_plugin.h",
|
||||
"+printing/buildflags/buildflags.h",
|
||||
"+third_party/blink/public",
|
||||
"+third_party/skia/include/core",
|
||||
"+ui/accessibility",
|
||||
"+ui/base",
|
||||
]
|
||||
|
@ -228,6 +228,7 @@ if (enable_pdf) {
|
||||
|
||||
deps = [
|
||||
"//base",
|
||||
"//skia",
|
||||
"//ui/gfx/geometry",
|
||||
]
|
||||
}
|
||||
|
@ -76,8 +76,12 @@ AccessibilityImageInfo::AccessibilityImageInfo() = default;
|
||||
|
||||
AccessibilityImageInfo::AccessibilityImageInfo(const std::string& alt_text,
|
||||
uint32_t text_run_index,
|
||||
const gfx::RectF& bounds)
|
||||
: alt_text(alt_text), text_run_index(text_run_index), bounds(bounds) {}
|
||||
const gfx::RectF& bounds,
|
||||
const SkBitmap& image_data)
|
||||
: alt_text(alt_text),
|
||||
text_run_index(text_run_index),
|
||||
bounds(bounds),
|
||||
image_data(image_data) {}
|
||||
|
||||
AccessibilityImageInfo::AccessibilityImageInfo(
|
||||
const AccessibilityImageInfo& other) = default;
|
||||
|
@ -10,6 +10,7 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "third_party/skia/include/core/SkBitmap.h"
|
||||
#include "ui/gfx/geometry/point.h"
|
||||
#include "ui/gfx/geometry/rect.h"
|
||||
#include "ui/gfx/geometry/rect_f.h"
|
||||
@ -130,18 +131,25 @@ struct AccessibilityImageInfo {
|
||||
AccessibilityImageInfo();
|
||||
AccessibilityImageInfo(const std::string& alt_text,
|
||||
uint32_t text_run_index,
|
||||
const gfx::RectF& bounds);
|
||||
const gfx::RectF& bounds,
|
||||
const SkBitmap& image_data);
|
||||
AccessibilityImageInfo(const AccessibilityImageInfo& other);
|
||||
~AccessibilityImageInfo();
|
||||
|
||||
// Alternate text for the image provided by PDF.
|
||||
std::string alt_text;
|
||||
|
||||
// We anchor the image to a char index, this denotes the text run before
|
||||
// which the image should be inserted in the accessibility tree. The text run
|
||||
// at this index should contain the anchor char index.
|
||||
uint32_t text_run_index = 0;
|
||||
|
||||
// Bounding box of the image.
|
||||
gfx::RectF bounds;
|
||||
|
||||
// Only populated if `alt_text` is empty or unavailable, and if the user has
|
||||
// requested that the OCR service tag the PDF so that it is made accessible.
|
||||
SkBitmap image_data;
|
||||
};
|
||||
|
||||
struct AccessibilityHighlightInfo {
|
||||
|
@ -391,7 +391,10 @@ class PDFEngine {
|
||||
int page_index,
|
||||
const std::vector<AccessibilityTextRunInfo>& text_runs) = 0;
|
||||
// For all the images in page `page_index`, get their alt texts and bounding
|
||||
// boxes.
|
||||
// boxes. If the alt text is empty or unavailable, and if the user has
|
||||
// requested that the OCR service tag the PDF so that it is made accessible,
|
||||
// transfer the raw image pixels in the `image_data` field. Otherwise do not
|
||||
// populate the `image_data` field.
|
||||
virtual std::vector<AccessibilityImageInfo> GetImageInfo(
|
||||
int page_index,
|
||||
uint32_t text_run_count) = 0;
|
||||
|
@ -1,5 +1,6 @@
|
||||
include_rules = [
|
||||
"+components/services/font/public/cpp",
|
||||
"+third_party/pdfium/public",
|
||||
"+ui/accessibility",
|
||||
"+ui/gfx/codec",
|
||||
]
|
||||
|
@ -114,9 +114,9 @@ TEST_F(AccessibilityTest, GetAccessibilityPage) {
|
||||
|
||||
TEST_F(AccessibilityTest, GetAccessibilityImageInfo) {
|
||||
static const AccessibilityImageInfo kExpectedImageInfo[] = {
|
||||
{"Image 1", 0, {380, 78, 67, 68}},
|
||||
{"Image 2", 0, {380, 385, 27, 28}},
|
||||
{"Image 3", 0, {380, 678, 1, 1}}};
|
||||
{"Image 1", 0, {380, 78, 67, 68}, {}},
|
||||
{"Image 2", 0, {380, 385, 27, 28}, {}},
|
||||
{"Image 3", 0, {380, 678, 1, 1}, {}}};
|
||||
|
||||
TestClient client;
|
||||
std::unique_ptr<PDFiumEngine> engine =
|
||||
|
@ -31,6 +31,9 @@
|
||||
#include "third_party/pdfium/public/cpp/fpdf_scopers.h"
|
||||
#include "third_party/pdfium/public/fpdf_annot.h"
|
||||
#include "third_party/pdfium/public/fpdf_catalog.h"
|
||||
#include "third_party/skia/include/core/SkImageInfo.h"
|
||||
#include "third_party/skia/include/core/SkPixmap.h"
|
||||
#include "ui/accessibility/accessibility_features.h"
|
||||
#include "ui/gfx/geometry/point.h"
|
||||
#include "ui/gfx/geometry/point_f.h"
|
||||
#include "ui/gfx/geometry/rect.h"
|
||||
@ -683,6 +686,7 @@ std::vector<AccessibilityImageInfo> PDFiumPage::GetImageInfo(
|
||||
cur_info.bounds =
|
||||
gfx::RectF(image.bounding_rect.x(), image.bounding_rect.y(),
|
||||
image.bounding_rect.width(), image.bounding_rect.height());
|
||||
cur_info.image_data = image.image_data;
|
||||
image_info.push_back(std::move(cur_info));
|
||||
}
|
||||
return image_info;
|
||||
@ -1160,6 +1164,7 @@ void PDFiumPage::CalculateImages() {
|
||||
continue;
|
||||
|
||||
Image image;
|
||||
image.page_object_index = i;
|
||||
image.bounding_rect = PageToScreen(gfx::Point(), 1.0, left, top, right,
|
||||
bottom, PageOrientation::kOriginal);
|
||||
|
||||
@ -1182,6 +1187,33 @@ void PDFiumPage::CalculateImages() {
|
||||
|
||||
if (!marked_content_id_image_map.empty())
|
||||
PopulateImageAltText(marked_content_id_image_map);
|
||||
|
||||
if (!features::IsPdfOcrEnabled())
|
||||
return;
|
||||
|
||||
// If requested by the user, we store the raw image data so that the OCR
|
||||
// service can try and retrieve textual and layout information from the image.
|
||||
// This is because alt text might be empty, or the PDF might simply be
|
||||
// untagged for accessibility.
|
||||
for (Image& image : images_) {
|
||||
if (!image.alt_text.empty())
|
||||
continue;
|
||||
|
||||
FPDF_PAGEOBJECT page_object =
|
||||
FPDFPage_GetObject(page, image.page_object_index);
|
||||
ScopedFPDFBitmap bitmap(
|
||||
FPDFImageObj_GetRenderedBitmap(engine_->doc(), page, page_object));
|
||||
if (!bitmap)
|
||||
continue;
|
||||
|
||||
SkImageInfo info = SkImageInfo::Make(
|
||||
FPDFBitmap_GetWidth(bitmap.get()), FPDFBitmap_GetHeight(bitmap.get()),
|
||||
kBGRA_8888_SkColorType, kOpaque_SkAlphaType);
|
||||
const size_t row_bytes = FPDFBitmap_GetStride(bitmap.get());
|
||||
SkPixmap pixels(info, FPDFBitmap_GetBuffer(bitmap.get()), row_bytes);
|
||||
if (image.image_data.tryAllocPixels(info, row_bytes))
|
||||
image.image_data.writePixels(pixels);
|
||||
}
|
||||
}
|
||||
|
||||
void PDFiumPage::PopulateImageAltText(
|
||||
|
@ -21,6 +21,7 @@
|
||||
#include "third_party/pdfium/public/fpdf_doc.h"
|
||||
#include "third_party/pdfium/public/fpdf_formfill.h"
|
||||
#include "third_party/pdfium/public/fpdf_text.h"
|
||||
#include "third_party/skia/include/core/SkBitmap.h"
|
||||
#include "ui/gfx/geometry/point_f.h"
|
||||
#include "ui/gfx/geometry/rect.h"
|
||||
|
||||
@ -75,8 +76,11 @@ class PDFiumPage {
|
||||
// bounding boxes.
|
||||
std::vector<AccessibilityLinkInfo> GetLinkInfo(
|
||||
const std::vector<AccessibilityTextRunInfo>& text_runs);
|
||||
|
||||
// For all the images on the page, get their alt texts and bounding boxes.
|
||||
// For all the images on the page, get their alt texts and bounding boxes. If
|
||||
// the alt text is empty or unavailable, and if the user has requested that
|
||||
// the OCR service tag the PDF so that it is made accessible, transfer the raw
|
||||
// image pixels in the `image_data` field. Otherwise do not populate the
|
||||
// `image_data` field.
|
||||
std::vector<AccessibilityImageInfo> GetImageInfo(uint32_t text_run_count);
|
||||
|
||||
// For all the highlights on the page, get their underlying text ranges and
|
||||
@ -220,6 +224,7 @@ class PDFiumPage {
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageHighlightTest, PopulateHighlights);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, CalculateImages);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageTest, ImageAltText);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageImageDataTest, ImageData);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, AnnotLinkGeneration);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, GetLinkTarget);
|
||||
FRIEND_TEST_ALL_PREFIXES(PDFiumPageLinkTest, LinkGeneration);
|
||||
@ -256,9 +261,13 @@ class PDFiumPage {
|
||||
Image(const Image& other);
|
||||
~Image();
|
||||
|
||||
gfx::Rect bounding_rect;
|
||||
// Alt text is available only for tagged PDFs.
|
||||
int page_object_index;
|
||||
// Alt text is available only for PDFs that are tagged for accessibility.
|
||||
std::string alt_text;
|
||||
gfx::Rect bounding_rect;
|
||||
// Image data is only stored if the user has requested that the OCR service
|
||||
// try to retrieve textual and layout information from this image.
|
||||
SkBitmap image_data;
|
||||
};
|
||||
|
||||
// Represents a highlight within the page.
|
||||
|
@ -11,6 +11,7 @@
|
||||
#include "base/files/file_path.h"
|
||||
#include "base/strings/string_util.h"
|
||||
#include "base/strings/stringprintf.h"
|
||||
#include "base/test/scoped_feature_list.h"
|
||||
#include "cc/test/pixel_comparator.h"
|
||||
#include "cc/test/pixel_test_utils.h"
|
||||
#include "pdf/accessibility_structs.h"
|
||||
@ -23,6 +24,7 @@
|
||||
#include "third_party/abseil-cpp/absl/types/optional.h"
|
||||
#include "third_party/pdfium/public/fpdf_formfill.h"
|
||||
#include "third_party/skia/include/core/SkBitmap.h"
|
||||
#include "ui/accessibility/accessibility_features.h"
|
||||
#include "ui/gfx/geometry/rect.h"
|
||||
#include "ui/gfx/geometry/rect_f.h"
|
||||
#include "ui/gfx/geometry/size_f.h"
|
||||
@ -283,6 +285,39 @@ TEST_F(PDFiumPageImageTest, ImageAltText) {
|
||||
EXPECT_EQ("", page.images_[2].alt_text);
|
||||
}
|
||||
|
||||
class PDFiumPageImageDataTest : public PDFiumPageImageTest {
|
||||
public:
|
||||
PDFiumPageImageDataTest() : enable_pdf_ocr_({features::kPdfOcr}) {}
|
||||
|
||||
PDFiumPageImageDataTest(const PDFiumPageImageDataTest&) = delete;
|
||||
PDFiumPageImageDataTest& operator=(const PDFiumPageImageDataTest&) = delete;
|
||||
~PDFiumPageImageDataTest() override = default;
|
||||
|
||||
private:
|
||||
base::test::ScopedFeatureList enable_pdf_ocr_;
|
||||
};
|
||||
|
||||
TEST_F(PDFiumPageImageDataTest, ImageData) {
|
||||
TestClient client;
|
||||
std::unique_ptr<PDFiumEngine> engine =
|
||||
InitializeEngine(&client, FILE_PATH_LITERAL("text_with_image.pdf"));
|
||||
ASSERT_TRUE(engine);
|
||||
ASSERT_EQ(1, engine->GetNumberOfPages());
|
||||
|
||||
PDFiumPage& page = GetPDFiumPageForTest(*engine, 0);
|
||||
page.CalculateImages();
|
||||
ASSERT_EQ(3u, page.images_.size());
|
||||
|
||||
ASSERT_FALSE(page.images_[0].alt_text.empty());
|
||||
EXPECT_TRUE(page.images_[0].image_data.drawsNothing());
|
||||
EXPECT_EQ(page.images_[0].image_data.width(), 0);
|
||||
EXPECT_EQ(page.images_[0].image_data.height(), 0);
|
||||
|
||||
ASSERT_TRUE(page.images_[2].alt_text.empty());
|
||||
EXPECT_EQ(page.images_[1].image_data.width(), 20);
|
||||
EXPECT_EQ(page.images_[1].image_data.height(), 20);
|
||||
}
|
||||
|
||||
using PDFiumPageTextTest = PDFiumTestBase;
|
||||
|
||||
TEST_F(PDFiumPageTextTest, TextRunBounds) {
|
||||
|
Reference in New Issue
Block a user