[PDF Ink Signatures] Load Ink data from PDFs

Give PdfInkModule the ability to call its client to load Ink data from
PDFs. PdfViewWebPlugin, the production client, asks PDFiumEngine to do
this. PDFiumEngine, in turn, uses code from pdfium_ink_reader.h to
fulfill the request.

The Ink data from PDFs load as ink::ModeledShape objects. Each object
gets a unique ID. PDFiumEngine stores an ID to FPDF_PAGEOBJECT mapping,
while PdfInkModule gets back an ID to ink::ModeledShape mapping,
bucketed by page index.

For now, just do the plumbing and store the data in PdfInkModule
using LoadedV2ShapeState, which is essentially the same as
FinishedStrokeState, but for shapes. In the near future, PdfInkModule
will use its data structure to perform hit tests to determine if the
paths in the PDF should be erased. It will coordinate with
PDFiumEngine to erase the objects in the PDF.

Bug: 353942910
Change-Id: I2fe6fb19d1e0fba9147eca1688d8a30e3695e9c4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5997152
Reviewed-by: Andy Phan <andyphan@chromium.org>
Code-Coverage: findit-for-me@appspot.gserviceaccount.com <findit-for-me@appspot.gserviceaccount.com>
Commit-Queue: Lei Zhang <thestig@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1380769}

pdf/pdf_ink_ids.h

@@ -14,6 +14,9 @@
 
 namespace chrome_pdf {
 
+// Identifies ink::ModeledShape objects.
+using InkModeledShapeId = base::StrongAlias<class InkModeledShapeIdTag, size_t>;
+
 // Identifies ink::Stroke objects.
 using InkStrokeId = base::StrongAlias<class InkStrokeIdTag, size_t>;
 

pdf/pdf_ink_module.cc

@@ -745,6 +745,18 @@ void PdfInkModule::HandleSetAnnotationModeMessage(
     const base::Value::Dict& message) {
   enabled_ = message.FindBool("enable").value();
   client_->OnAnnotationModeToggled(enabled_);
+  if (enabled_ && !loaded_data_from_pdf_) {
+    loaded_data_from_pdf_ = true;
+    PdfInkModuleClient::DocumentV2InkPathShapesMap loaded_v2_shapes =
+        client_->LoadV2InkPathsFromPdf();
+    for (auto& [page_index, page_shape_map] : loaded_v2_shapes) {
+      PageV2InkPathShapes& page_shapes = loaded_v2_shapes_[page_index];
+      page_shapes.reserve(page_shape_map.size());
+      for (auto& [shape_id, shape] : page_shape_map) {
+        page_shapes.emplace_back(shape, shape_id);
+      }
+    }
+  }
   MaybeSetCursor();
 }
 
@@ -988,6 +1000,18 @@ PdfInkModule::FinishedStrokeState& PdfInkModule::FinishedStrokeState::operator=(
 
 PdfInkModule::FinishedStrokeState::~FinishedStrokeState() = default;
 
+PdfInkModule::LoadedV2ShapeState::LoadedV2ShapeState(ink::ModeledShape shape,
+                                                     InkModeledShapeId id)
+    : shape(std::move(shape)), id(id) {}
+
+PdfInkModule::LoadedV2ShapeState::LoadedV2ShapeState(
+    PdfInkModule::LoadedV2ShapeState&&) noexcept = default;
+
+PdfInkModule::LoadedV2ShapeState& PdfInkModule::LoadedV2ShapeState::operator=(
+    PdfInkModule::LoadedV2ShapeState&&) noexcept = default;
+
+PdfInkModule::LoadedV2ShapeState::~LoadedV2ShapeState() = default;
+
 PdfInkModule::StrokeIdGenerator::StrokeIdGenerator() = default;
 
 PdfInkModule::StrokeIdGenerator::~StrokeIdGenerator() = default;

pdf/pdf_ink_module.h

@@ -12,6 +12,7 @@
 #include <vector>
 
 #include "base/containers/flat_set.h"
+#include "base/gtest_prod_util.h"
 #include "base/memory/raw_ref.h"
 #include "base/time/time.h"
 #include "base/values.h"
@@ -20,6 +21,7 @@
 #include "pdf/pdf_ink_ids.h"
 #include "pdf/pdf_ink_undo_redo_model.h"
 #include "third_party/abseil-cpp/absl/types/variant.h"
+#include "third_party/ink/src/ink/geometry/modeled_shape.h"
 #include "third_party/ink/src/ink/strokes/in_progress_stroke.h"
 #include "third_party/ink/src/ink/strokes/input/stroke_input_batch.h"
 #include "third_party/ink/src/ink/strokes/stroke.h"
@@ -168,6 +170,34 @@ class PdfInkModule {
       const;
 
  private:
+  FRIEND_TEST_ALL_PREFIXES(PdfInkModuleTest, HandleSetAnnotationModeMessage);
+
+  // A shape that was loaded from a "V2" path from the PDF itself, its ID, and
+  // whether it should be drawn or not.
+  struct LoadedV2ShapeState {
+    LoadedV2ShapeState(ink::ModeledShape shape, InkModeledShapeId id);
+    LoadedV2ShapeState(const LoadedV2ShapeState&) = delete;
+    LoadedV2ShapeState& operator=(const LoadedV2ShapeState&) = delete;
+    LoadedV2ShapeState(LoadedV2ShapeState&&) noexcept;
+    LoadedV2ShapeState& operator=(LoadedV2ShapeState&&) noexcept;
+    ~LoadedV2ShapeState();
+
+    // Coordinates for each shape are stored in a canonical format specified in
+    // pdf_ink_transform.h.
+    ink::ModeledShape shape;
+
+    // A unique ID to identify this shape.
+    InkModeledShapeId id;
+
+    bool should_draw = true;
+  };
+
+  // Like PageStrokes, but for shapes created from "V2" paths in the PDF.
+  using PageV2InkPathShapes = std::vector<LoadedV2ShapeState>;
+
+  // Like DocumentStrokesMap, but for PageV2InkPathShapes.
+  using DocumentV2InkPathShapesMap = std::map<int, PageV2InkPathShapes>;
+
   struct DrawingStrokeState {
     DrawingStrokeState();
     DrawingStrokeState(const DrawingStrokeState&) = delete;
@@ -307,6 +337,11 @@ class PdfInkModule {
 
   bool enabled_ = false;
 
+  bool loaded_data_from_pdf_ = false;
+
+  // Shapes loaded from the PDF.
+  DocumentV2InkPathShapesMap loaded_v2_shapes_;
+
   // Generates IDs for use in FinishedStrokeState and PdfInkUndoRedoModel.
   StrokeIdGenerator stroke_id_generator_;
 

pdf/pdf_ink_module_client.h

@@ -5,9 +5,12 @@
 #ifndef PDF_PDF_INK_MODULE_CLIENT_H_
 #define PDF_PDF_INK_MODULE_CLIENT_H_
 
+#include <map>
+
 #include "pdf/buildflags.h"
 #include "pdf/page_orientation.h"
 #include "pdf/pdf_ink_ids.h"
+#include "third_party/ink/src/ink/geometry/modeled_shape.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/vector2d.h"
@@ -26,6 +29,14 @@ namespace chrome_pdf {
 
 class PdfInkModuleClient {
  public:
+  // Key: ID to identify a shape.
+  // Value: The Ink shape.
+  using PageV2InkPathShapesMap = std::map<InkModeledShapeId, ink::ModeledShape>;
+
+  // Key: 0-based page index.
+  // Value: Map of shapes on the page.
+  using DocumentV2InkPathShapesMap = std::map<int, PageV2InkPathShapesMap>;
+
   virtual ~PdfInkModuleClient() = default;
 
   // Gets the current page orientation.
@@ -54,6 +65,9 @@ class PdfInkModuleClient {
   // Returns whether the page at `page_index` is visible or not.
   virtual bool IsPageVisible(int page_index) = 0;
 
+  // Asks the client to load Ink data from the PDF.
+  virtual DocumentV2InkPathShapesMap LoadV2InkPathsFromPdf() = 0;
+
   // Notifies the client whether annotation mode is enabled or not.
   virtual void OnAnnotationModeToggled(bool enable) {}
 

pdf/pdf_ink_module_unittest.cc

@@ -46,9 +46,11 @@
 using testing::_;
 using testing::ElementsAre;
 using testing::ElementsAreArray;
+using testing::Field;
 using testing::InSequence;
 using testing::Pair;
 using testing::Pointwise;
+using testing::Return;
 using testing::SizeIs;
 
 namespace chrome_pdf {
@@ -204,6 +206,11 @@ class FakeClient : public PdfInkModuleClient {
     return base::Contains(visible_page_indices_, page_index);
   }
 
+  MOCK_METHOD(PdfInkModuleClient::DocumentV2InkPathShapesMap,
+              LoadV2InkPathsFromPdf,
+              (),
+              (override));
+
   MOCK_METHOD(void, PostMessage, (base::Value::Dict message), (override));
 
   MOCK_METHOD(void,
@@ -299,6 +306,8 @@ class PdfInkModuleTest : public testing::Test {
   PdfInkModule ink_module_{client_};
 };
 
+}  // namespace
+
 TEST_F(PdfInkModuleTest, UnknownMessage) {
   base::Value::Dict message;
   message.Set("type", "nonInkMessage");
@@ -529,6 +538,25 @@ TEST_F(PdfInkModuleTest, HandleSetAnnotationBrushMessageColorZero) {
 }
 
 TEST_F(PdfInkModuleTest, HandleSetAnnotationModeMessage) {
+  EXPECT_CALL(client(), LoadV2InkPathsFromPdf())
+      .WillOnce(Return(PdfInkModuleClient::DocumentV2InkPathShapesMap{
+          {0,
+           PdfInkModuleClient::PageV2InkPathShapesMap{
+               {InkModeledShapeId(0), ink::ModeledShape()},
+               {InkModeledShapeId(1), ink::ModeledShape()}}},
+          {3,
+           PdfInkModuleClient::PageV2InkPathShapesMap{
+               {InkModeledShapeId(2), ink::ModeledShape()}}},
+      }));
+
+  const auto kShapeMapMatcher = ElementsAre(
+      Pair(0, ElementsAre(Field(&PdfInkModule::LoadedV2ShapeState::id,
+                                InkModeledShapeId(0)),
+                          Field(&PdfInkModule::LoadedV2ShapeState::id,
+                                InkModeledShapeId(1)))),
+      Pair(3, ElementsAre(Field(&PdfInkModule::LoadedV2ShapeState::id,
+                                InkModeledShapeId(2)))));
+
   EXPECT_FALSE(ink_module().enabled());
 
   base::Value::Dict message =
@@ -536,14 +564,17 @@ TEST_F(PdfInkModuleTest, HandleSetAnnotationModeMessage) {
 
   EXPECT_TRUE(ink_module().OnMessage(message));
   EXPECT_FALSE(ink_module().enabled());
+  EXPECT_TRUE(ink_module().loaded_v2_shapes_.empty());
 
   message.Set("enable", true);
   EXPECT_TRUE(ink_module().OnMessage(message));
   EXPECT_TRUE(ink_module().enabled());
+  EXPECT_THAT(ink_module().loaded_v2_shapes_, kShapeMapMatcher);
 
   message.Set("enable", false);
   EXPECT_TRUE(ink_module().OnMessage(message));
   EXPECT_FALSE(ink_module().enabled());
+  EXPECT_THAT(ink_module().loaded_v2_shapes_, kShapeMapMatcher);
 }
 
 TEST_F(PdfInkModuleTest, MaybeSetCursorWhenTogglingAnnotationMode) {
@@ -1729,6 +1760,4 @@ TEST_F(PdfInkModuleGetVisibleStrokesTest, MultiplePageStrokes) {
                             {expected_page1_horz_line_input_batch.value()}))));
 }
 
-}  // namespace
-
 }  // namespace chrome_pdf

pdf/pdf_view_web_plugin.cc

@@ -297,6 +297,22 @@ class PdfViewWebPlugin::PdfInkModuleClientImpl : public PdfInkModuleClient {
     return plugin_->engine_->IsPageVisible(page_index);
   }
 
+  DocumentV2InkPathShapesMap LoadV2InkPathsFromPdf() override {
+    DocumentV2InkPathShapesMap shapes_map;
+
+    for (int i = 0; i < plugin_->engine_->GetNumberOfPages(); ++i) {
+      PageV2InkPathShapesMap page_shapes_map =
+          plugin_->engine_->LoadV2InkPathsForPage(i);
+      if (page_shapes_map.empty()) {
+        continue;
+      }
+
+      shapes_map[i] = std::move(page_shapes_map);
+    }
+
+    return shapes_map;
+  }
+
   void OnAnnotationModeToggled(bool enable) override {
     plugin_->engine_->SetFormHighlight(/*enable_form=*/!enable);
     if (enable) {

pdf/pdf_view_web_plugin_unittest.cc

@@ -108,6 +108,7 @@ using ::testing::IsFalse;
 using ::testing::IsTrue;
 using ::testing::MockFunction;
 using ::testing::NiceMock;
+using ::testing::Pair;
 using ::testing::Pointwise;
 using ::testing::Return;
 using ::testing::SaveArg;
@@ -2517,6 +2518,40 @@ TEST_F(PdfViewWebPluginInkTest, Invalidate) {
   EXPECT_EQ(3u, plugin_->deferred_invalidates_for_testing().size());
 }
 
+TEST_F(PdfViewWebPluginInkTest, LoadV2InkPathsForPage) {
+  const std::map<InkModeledShapeId, ink::ModeledShape> kEmptyMap;
+  const std::map<InkModeledShapeId, ink::ModeledShape> kMap0{
+      {InkModeledShapeId(0), ink::ModeledShape()},
+  };
+  const std::map<InkModeledShapeId, ink::ModeledShape> kMap1{
+      {InkModeledShapeId(1), ink::ModeledShape()},
+      {InkModeledShapeId(2), ink::ModeledShape()},
+  };
+  const std::map<InkModeledShapeId, ink::ModeledShape> kMap2{
+      {InkModeledShapeId(3), ink::ModeledShape()},
+      {InkModeledShapeId(4), ink::ModeledShape()},
+      {InkModeledShapeId(5), ink::ModeledShape()},
+  };
+
+  EXPECT_CALL(*engine_ptr_, LoadV2InkPathsForPage(testing::Lt(12)))
+      .Times(12)
+      .WillOnce(Return(kMap0))
+      .WillOnce(Return(kMap1))
+      .WillRepeatedly(Return(kEmptyMap));
+  EXPECT_CALL(*engine_ptr_, LoadV2InkPathsForPage(12)).WillOnce(Return(kMap2));
+
+  const PdfInkModuleClient::DocumentV2InkPathShapesMap result =
+      plugin_->ink_module_client_for_testing()->LoadV2InkPathsFromPdf();
+  EXPECT_THAT(
+      result,
+      ElementsAre(Pair(0, ElementsAre(Pair(InkModeledShapeId(0), _))),
+                  Pair(1, ElementsAre(Pair(InkModeledShapeId(1), _),
+                                      Pair(InkModeledShapeId(2), _))),
+                  Pair(12, ElementsAre(Pair(InkModeledShapeId(3), _),
+                                       Pair(InkModeledShapeId(4), _),
+                                       Pair(InkModeledShapeId(5), _)))));
+}
+
 TEST_F(PdfViewWebPluginInkTest, SendThumbnailUpdatesInkThumbnail) {
   SetUpWithTrivialInkStrokes();
 

pdf/pdfium/pdfium_engine.cc

@@ -24,6 +24,7 @@
 #include "base/check_op.h"
 #include "base/containers/contains.h"
 #include "base/containers/flat_map.h"
+#include "base/dcheck_is_on.h"
 #include "base/feature_list.h"
 #include "base/functional/bind.h"
 #include "base/location.h"
@@ -95,6 +96,7 @@
 #endif
 
 #if BUILDFLAG(ENABLE_PDF_INK2)
+#include "pdf/pdfium/pdfium_ink_reader.h"
 #include "pdf/pdfium/pdfium_ink_writer.h"
 #include "third_party/ink/src/ink/strokes/stroke.h"
 #endif
@@ -4419,6 +4421,29 @@ void PDFiumEngine::UpdateStrokeActive(int page_index,
   CHECK(result);
   ink_stroked_pages_needing_regeneration_.insert(page_index);
 }
+
+std::map<InkModeledShapeId, ink::ModeledShape>
+PDFiumEngine::LoadV2InkPathsForPage(int page_index) {
+  CHECK(PageIndexInBounds(page_index));
+
+#if DCHECK_IS_ON()
+  const bool inserted =
+      pages_with_loaded_v2_ink_paths_.insert(page_index).second;
+  CHECK(inserted);
+#endif  // DCHECK_IS_ON()
+
+  std::map<InkModeledShapeId, ink::ModeledShape> page_shape_map;
+
+  std::vector<ReadV2InkPathResult> read_results =
+      ReadV2InkPathsFromPageAsModeledShapes(pages_[page_index]->GetPage());
+  for (auto& read_result : read_results) {
+    InkModeledShapeId id(next_ink_modeled_shape_id_++);
+    page_shape_map[id] = std::move(read_result.shape);
+    ink_modeled_shape_map_[id] = read_result.page_object;
+  }
+
+  return page_shape_map;
+}
 #endif  // BUILDFLAG(ENABLE_PDF_INK2)
 
 PDFiumEngine::ProgressivePaint::ProgressivePaint(int index,

pdf/pdfium/pdfium_engine.h

@@ -17,6 +17,7 @@
 
 #include "base/containers/flat_map.h"
 #include "base/containers/span.h"
+#include "base/dcheck_is_on.h"
 #include "base/functional/callback.h"
 #include "base/memory/raw_ptr.h"
 #include "base/memory/raw_span.h"
@@ -61,6 +62,7 @@
 
 #if BUILDFLAG(ENABLE_PDF_INK2)
 #include "pdf/pdf_ink_ids.h"
+#include "third_party/ink/src/ink/geometry/modeled_shape.h"
 #endif
 
 #if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
@@ -383,6 +385,27 @@ class PDFiumEngine : public DocumentLoader::Client, public IFSDK_PAUSE {
   // rendering or saving out to PDF data.  Their inclusion can be restored if
   // another call makes them active again.  Virtual to support testing.
   virtual void UpdateStrokeActive(int page_index, InkStrokeId id, bool active);
+
+  // Loads "V2" Ink paths from a page in the PDF identified by `page_index`. The
+  // `page_index` must be in bounds.
+  //
+  // Returns a mapping to identify shapes by IDs. In `this`, store a mapping
+  // from IDs to PDFium page objects. This allows the caller to associate shapes
+  // with their corresponding PDFium page objects, without any direct exposure
+  // to PDFium types.
+  //
+  // It is the caller's responsibility to not call this multiple times per page,
+  // or else there will be multiple IDs associated with the same underlying
+  // PDFium page object.
+  //
+  // Virtual to support testing.
+  virtual std::map<InkModeledShapeId, ink::ModeledShape> LoadV2InkPathsForPage(
+      int page_index);
+
+  const std::map<InkModeledShapeId, FPDF_PAGEOBJECT>&
+  ink_modeled_shape_map_for_testing() const {
+    return ink_modeled_shape_map_;
+  }
 #endif  // BUILDFLAG(ENABLE_PDF_INK2)
 
   // DocumentLoader::Client:
@@ -1124,6 +1147,23 @@ class PDFiumEngine : public DocumentLoader::Client, public IFSDK_PAUSE {
 
   PDFiumPrint print_;
 
+#if BUILDFLAG(ENABLE_PDF_INK2)
+#if DCHECK_IS_ON()
+  // Used to keep track of LoadV2InkPathsForPage() calls as a sanity check.
+  // Stores the 0-based page indices for pages that have been loaded.
+  std::set<int> pages_with_loaded_v2_ink_paths_;
+#endif  // DCHECK_IS_ON()
+
+  // Used to hand out unique IDs of type InkModeledShapeId for the V2 Ink paths
+  // read out of the PDF. It is stored here as the raw type to simplify
+  // management.
+  size_t next_ink_modeled_shape_id_ = 0;
+
+  // Key: ID to identify a shape.
+  // Value: The PDFium page object associated with the shape.
+  std::map<InkModeledShapeId, FPDF_PAGEOBJECT> ink_modeled_shape_map_;
+#endif  // BUILDFLAG(ENABLE_PDF_INK2)
+
   base::WeakPtrFactory<PDFiumEngine> weak_factory_{this};
 
   // Weak pointers from this factory are used to bind the ContinueFind()

pdf/pdfium/pdfium_engine_unittest.cc

@@ -2003,9 +2003,9 @@ TEST_P(PDFiumEngineReadOnlyTest, UnselectText) {
 INSTANTIATE_TEST_SUITE_P(All, PDFiumEngineReadOnlyTest, testing::Bool());
 
 #if BUILDFLAG(ENABLE_PDF_INK2)
-using PDFiumEngineAnnotationModeTest = PDFiumTestBase;
+using PDFiumEngineInkTest = PDFiumTestBase;
 
-TEST_P(PDFiumEngineAnnotationModeTest, KillFormFocus) {
+TEST_P(PDFiumEngineInkTest, KillFormFocusInAnnotationMode) {
   NiceMock<MockTestClient> client;
   std::unique_ptr<PDFiumEngine> engine = InitializeEngine(
       &client, FILE_PATH_LITERAL("annotation_form_fields.pdf"));
@@ -2020,7 +2020,7 @@ TEST_P(PDFiumEngineAnnotationModeTest, KillFormFocus) {
   engine->UpdateFocus(true);
 }
 
-TEST_P(PDFiumEngineAnnotationModeTest, CannotSelectText) {
+TEST_P(PDFiumEngineInkTest, CannotSelectTextInAnnotationMode) {
   NiceMock<MockTestClient> client;
   std::unique_ptr<PDFiumEngine> engine =
       InitializeEngine(&client, FILE_PATH_LITERAL("hello_world2.pdf"));
@@ -2038,7 +2038,30 @@ TEST_P(PDFiumEngineAnnotationModeTest, CannotSelectText) {
   EXPECT_THAT(engine->GetSelectedText(), IsEmpty());
 }
 
-INSTANTIATE_TEST_SUITE_P(All, PDFiumEngineAnnotationModeTest, testing::Bool());
+TEST_P(PDFiumEngineInkTest, LoadV2InkPathsForPage) {
+  NiceMock<MockTestClient> client;
+  std::unique_ptr<PDFiumEngine> engine =
+      InitializeEngine(&client, FILE_PATH_LITERAL("ink_v2.pdf"));
+  ASSERT_TRUE(engine);
+  ASSERT_EQ(1, engine->GetNumberOfPages());
+  EXPECT_TRUE(engine->ink_modeled_shape_map_for_testing().empty());
+
+  std::map<InkModeledShapeId, ink::ModeledShape> ink_shapes =
+      engine->LoadV2InkPathsForPage(/*page_index=*/0);
+  ASSERT_EQ(1u, ink_shapes.size());
+  const auto ink_shapes_it = ink_shapes.begin();
+
+  const std::map<InkModeledShapeId, FPDF_PAGEOBJECT>& pdf_shapes =
+      engine->ink_modeled_shape_map_for_testing();
+  ASSERT_EQ(1u, pdf_shapes.size());
+  const auto pdf_shapes_it = pdf_shapes.begin();
+
+  EXPECT_EQ(ink_shapes_it->first, pdf_shapes_it->first);
+  EXPECT_EQ(1u, ink_shapes_it->second.Meshes().size());
+  EXPECT_TRUE(pdf_shapes_it->second);
+}
+
+INSTANTIATE_TEST_SUITE_P(All, PDFiumEngineInkTest, testing::Bool());
 
 using PDFiumEngineInkStrokesTest = PDFiumTestBase;
 

pdf/test/test_pdfium_engine.h

@@ -106,7 +106,12 @@ class TestPDFiumEngine : public PDFiumEngine {
               (override));
 
   MOCK_METHOD(void, UpdateStrokeActive, (int, InkStrokeId, bool), (override));
-#endif
+
+  MOCK_METHOD((std::map<InkModeledShapeId, ink::ModeledShape>),
+              LoadV2InkPathsForPage,
+              (int),
+              (override));
+#endif  // BUILDFLAG(ENABLE_PDF_INK2)
 
   std::vector<uint8_t> GetSaveData() override;