media/gpu/vaapiVEA: Refactor the code of creating surfaces

This renames the variables of storing available VA surfaces and
cleans up the VA surface creation code for better readability.
After this CL, all surfaces required for EncodeJobs are created
before creating EncodeJobs.It enables avoiding the redundant
creation of EncodeJobs.

Bug: 1186051
Test: video.EncodeAccel.*
Change-Id: I04f4a95e061e30fc390b2350f979adc08e01b97a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3054270
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: Miguel Casas <mcasas@chromium.org>
Reviewed-by: Zhaoliang Ma <zhaoliang.ma@intel.com>
Cr-Commit-Position: refs/heads/main@{#914670}

media/gpu/vaapi/vaapi_video_encode_accelerator.cc

@@ -359,31 +359,14 @@ void VaapiVideoEncodeAccelerator::InitializeTask(const Config& config) {
       expected_input_coded_size_.width() <= encoder_->GetCodedSize().width() &&
       expected_input_coded_size_.height() <= encoder_->GetCodedSize().height());
 
-  va_surfaces_per_video_frame_ =
-      native_input_mode_
-          ?
-          // In native input mode, we do not need surfaces for input frames.
-          kNumSurfacesForOutputPicture
-          :
-          // In non-native mode, we need to create additional surfaces for input
-          // frames.
-          kNumSurfacesForOutputPicture + kNumSurfacesPerInputVideoFrame;
-
   // The number of required buffers is the number of required reference frames
   // + 1 for the current frame to be encoded.
   const size_t max_ref_frames = encoder_->GetMaxNumOfRefFrames();
   num_frames_in_flight_ = std::max(kMinNumFramesInFlight, max_ref_frames);
   DVLOGF(1) << "Frames in flight: " << num_frames_in_flight_;
 
-  // The surface size for the reconstructed surface (and input surface in non
-  // native input mode) is the coded size.
-  available_va_surfaces_ = vaapi_wrapper_->CreateContextAndScopedVASurfaces(
-      kVaSurfaceFormat, encoder_->GetCodedSize(),
-      {VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
-      (num_frames_in_flight_ + 1) * va_surfaces_per_video_frame_,
-      /*visible_size=*/absl::nullopt);
-  if (available_va_surfaces_.empty()) {
-    NOTIFY_ERROR(kPlatformFailureError, "Failed creating VASurfaces");
+  if (!vaapi_wrapper_->CreateContext(encoder_->GetCodedSize())) {
+    NOTIFY_ERROR(kPlatformFailureError, "Failed creating VAContext");
     return;
   }
 
@@ -421,6 +404,19 @@ void VaapiVideoEncodeAccelerator::RecycleVASurface(
   DVLOGF(4) << "va_surface_id: " << va_surface_id;
 
   va_surfaces->push_back(std::move(va_surface));
+
+  // At least one surface must available in each |available_encode_surfaces_|
+  // and |available_vpp_dest_surfaces_| to succeed in EncodePendingInputs().
+  // Checks here to avoid redundant EncodePendingInputs() call.
+  for (const auto& surfaces : available_encode_surfaces_) {
+    if (surfaces.second.empty())
+      return;
+  }
+  for (const auto& surfaces : available_vpp_dest_surfaces_) {
+    if (surfaces.second.empty())
+      return;
+  }
+
   EncodePendingInputs();
 }
 
@@ -537,80 +533,19 @@ VaapiVideoEncodeAccelerator::GetAvailableVASurfaceAsRefCounted(
                                          std::move(release_cb));
 }
 
-scoped_refptr<VASurface>
-VaapiVideoEncodeAccelerator::BlitSurfaceWithCreateVppIfNeeded(
-    const VASurface& input_surface,
-    const gfx::Rect& input_visible_rect,
-    const gfx::Size& encode_size,
-    size_t num_va_surfaces) {
-  DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
-
-  if (!vpp_vaapi_wrapper_) {
-    vpp_vaapi_wrapper_ = VaapiWrapper::Create(
-        VaapiWrapper::kVideoProcess, VAProfileNone,
-        EncryptionScheme::kUnencrypted,
-        base::BindRepeating(
-            &ReportVaapiErrorToUMA,
-            "Media.VaapiVideoEncodeAccelerator.Vpp.VAAPIError"));
-    if (!vpp_vaapi_wrapper_) {
-      NOTIFY_ERROR(kPlatformFailureError,
-                   "Failed to initialize VppVaapiWrapper");
-      return nullptr;
-    }
-
-    // VA context for VPP is not associated with a specific resolution.
-    if (!vpp_vaapi_wrapper_->CreateContext(gfx::Size())) {
-      NOTIFY_ERROR(kPlatformFailureError, "Failed creating Context for VPP");
-      return nullptr;
-    }
-  }
-
-  if (!base::Contains(available_vpp_va_surfaces_, encode_size)) {
-    auto scoped_va_surfaces = vpp_vaapi_wrapper_->CreateScopedVASurfaces(
-        kVaSurfaceFormat, encode_size,
-        {VaapiWrapper::SurfaceUsageHint::kVideoProcessWrite,
-         VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
-        num_va_surfaces, /*visible_size=*/absl::nullopt,
-        /*va_fourcc=*/absl::nullopt);
-    if (scoped_va_surfaces.empty()) {
-      NOTIFY_ERROR(kPlatformFailureError, "Failed creating VASurfaces");
-      return nullptr;
-    }
-
-    available_vpp_va_surfaces_[encode_size] = std::move(scoped_va_surfaces);
-  }
-
-  auto blit_surface = GetAvailableVASurfaceAsRefCounted(
-      &available_vpp_va_surfaces_[encode_size]);
-  if (!vpp_vaapi_wrapper_->BlitSurface(input_surface, *blit_surface,
-                                       input_visible_rect,
-                                       gfx::Rect(encode_size))) {
-    NOTIFY_ERROR(kPlatformFailureError,
-                 "Failed BlitSurface on frame size: "
-                     << input_surface.size().ToString()
-                     << " (visible rect: " << input_visible_rect.ToString()
-                     << ") -> encode size: " << encode_size.ToString());
-    return nullptr;
-  }
-
-  return blit_surface;
-}
-
 bool VaapiVideoEncodeAccelerator::CreateSurfacesForGpuMemoryBufferEncoding(
     const VideoFrame& frame,
-    const gfx::Size& encode_size,
-    scoped_refptr<VASurface>* input_surface,
-    scoped_refptr<VASurface>* reconstructed_surface) {
+    const std::vector<gfx::Size>& spatial_layer_resolutions,
+    std::vector<scoped_refptr<VASurface>>* input_surfaces,
+    std::vector<scoped_refptr<VASurface>>* reconstructed_surfaces) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
   DCHECK(native_input_mode_);
-
   if (frame.storage_type() != VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
     NOTIFY_ERROR(kPlatformFailureError,
                  "Unexpected storage: "
                      << VideoFrame::StorageTypeToString(frame.storage_type()));
     return false;
   }
-
   if (frame.format() != PIXEL_FORMAT_NV12) {
     NOTIFY_ERROR(
         kPlatformFailureError,
@@ -618,21 +553,6 @@ bool VaapiVideoEncodeAccelerator::CreateSurfacesForGpuMemoryBufferEncoding(
     return false;
   }
 
-  const bool do_vpp = frame.visible_rect() != gfx::Rect(encode_size);
-  if (do_vpp) {
-    constexpr size_t kNumSurfaces = 2;  // For input and reconstructed surface.
-    if (base::Contains(available_vpp_va_surfaces_, encode_size) &&
-        available_vpp_va_surfaces_[encode_size].size() < kNumSurfaces) {
-      DVLOGF(4) << "Not enough surfaces available";
-      return false;
-    }
-  } else {
-    if (available_va_surfaces_.empty()) {  // For the reconstructed surface.
-      DVLOGF(4) << "Not surface available";
-      return false;
-    }
-  }
-
   // Create VASurface from GpuMemory-based VideoFrame.
   scoped_refptr<gfx::NativePixmap> pixmap = CreateNativePixmapDmaBuf(&frame);
   if (!pixmap) {
@@ -641,35 +561,44 @@ bool VaapiVideoEncodeAccelerator::CreateSurfacesForGpuMemoryBufferEncoding(
     return false;
   }
 
-  *input_surface = vaapi_wrapper_->CreateVASurfaceForPixmap(std::move(pixmap));
-  if (!(*input_surface)) {
+  scoped_refptr<VASurface> source_surface =
+      vaapi_wrapper_->CreateVASurfaceForPixmap(std::move(pixmap));
+  if (!source_surface) {
     NOTIFY_ERROR(kPlatformFailureError, "Failed to create VASurface");
     return false;
   }
 
-  // Crop and Scale input surface to a surface whose size is |encode_size|.
-  // The size of a reconstructed surface is also |encode_size|.
-  if (do_vpp) {
-    // Create blit destination and reconstructed surfaces.
-    *input_surface = BlitSurfaceWithCreateVppIfNeeded(
-        *input_surface->get(), frame.visible_rect(), encode_size,
-        (num_frames_in_flight_ + 1) * 2);
-    if (!(*input_surface))
+  // Create input and reconstructed surfaces.
+  input_surfaces->reserve(spatial_layer_resolutions.size());
+  reconstructed_surfaces->reserve(spatial_layer_resolutions.size());
+  for (const gfx::Size& encode_size : spatial_layer_resolutions) {
+    const bool engage_vpp = frame.visible_rect() != gfx::Rect(encode_size);
+    // Crop and Scale input surface to a surface whose size is |encode_size|.
+    // The size of a reconstructed surface is also |encode_size|.
+    if (engage_vpp) {
+      auto blit_surface = ExecuteBlitSurface(*source_surface,
+                                             frame.visible_rect(), encode_size);
+      input_surfaces->push_back(std::move(blit_surface));
+    } else {
+      input_surfaces->emplace_back(source_surface);
+    }
+
+    if (!CreateSurfacesIfNeeded(*vaapi_wrapper_, available_encode_surfaces_,
+                                encode_size,
+                                {VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
+                                num_frames_in_flight_ + 1)) {
       return false;
-    // A reconstructed surface is fine to be created by VPP VaapiWrapper and
-    // encoder VaapiWrapper. Use one created by VPP VaapiWrapper when VPP is
-    // executed.
-    *reconstructed_surface = GetAvailableVASurfaceAsRefCounted(
-        &available_vpp_va_surfaces_[encode_size]);
-  } else {
-    // TODO(crbug.com/1186051): Create VA Surface here for the first time, not
-    // in Initialize()
-    *reconstructed_surface =
-        GetAvailableVASurfaceAsRefCounted(&available_va_surfaces_);
+    }
+    if (available_encode_surfaces_[encode_size].empty()) {
+      DVLOGF(4) << "Not enough reconstructed surface available";
+      return false;
+    }
+    reconstructed_surfaces->emplace_back(GetAvailableVASurfaceAsRefCounted(
+        &available_encode_surfaces_[encode_size]));
+    DCHECK(!!reconstructed_surfaces->back());
   }
 
-  DCHECK(*input_surface);
-  return !!(*reconstructed_surface);
+  return true;
 }
 
 bool VaapiVideoEncodeAccelerator::CreateSurfacesForShmemEncoding(
@@ -691,7 +620,7 @@ bool VaapiVideoEncodeAccelerator::CreateSurfacesForShmemEncoding(
     return false;
   }
 
-  DCHECK_EQ(visible_rect_.origin(), gfx::Point(0, 0));
+  DCHECK(visible_rect_.origin().IsOrigin());
   if (visible_rect_ != frame.visible_rect()) {
     // In non-zero copy mode, the client is responsible for scaling and
     // cropping.
@@ -702,38 +631,133 @@ bool VaapiVideoEncodeAccelerator::CreateSurfacesForShmemEncoding(
     return false;
   }
 
-  constexpr size_t kNumSurfaces = 2;  // input and reconstructed surface.
-  if (available_va_surfaces_.size() < kNumSurfaces) {
+  const gfx::Size& encode_size = encoder_->GetCodedSize();
+  constexpr size_t kNumSurfaces = 2;  // For input and reconstructed surface.
+  if (!CreateSurfacesIfNeeded(*vaapi_wrapper_, available_encode_surfaces_,
+                              encode_size,
+                              {VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
+                              (num_frames_in_flight_ + 1) * kNumSurfaces)) {
+    return false;
+  }
+
+  auto& surfaces = available_encode_surfaces_[encode_size];
+  if (surfaces.size() < kNumSurfaces) {
     DVLOGF(4) << "Not enough surfaces available";
     return false;
   }
 
-  *input_surface = GetAvailableVASurfaceAsRefCounted(&available_va_surfaces_);
-  *reconstructed_surface =
-      GetAvailableVASurfaceAsRefCounted(&available_va_surfaces_);
+  *input_surface = GetAvailableVASurfaceAsRefCounted(&surfaces);
+  *reconstructed_surface = GetAvailableVASurfaceAsRefCounted(&surfaces);
   return true;
 }
 
-std::unique_ptr<VaapiVideoEncoderDelegate::EncodeJob>
-VaapiVideoEncodeAccelerator::CreateEncodeJob(scoped_refptr<VideoFrame> frame,
-                                             bool force_keyframe,
-                                             const gfx::Size& encode_size) {
+bool VaapiVideoEncodeAccelerator::CreateSurfacesIfNeeded(
+    VaapiWrapper& vaapi_wrapper,
+    ScopedVASurfacesMap& scoped_surfaces_map,
+    const gfx::Size& encode_size,
+    const std::vector<VaapiWrapper::SurfaceUsageHint>& surface_usage_hints,
+    size_t num_surfaces) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
-  DCHECK(frame);
+  if (base::Contains(scoped_surfaces_map, encode_size))
+    return true;
 
-  scoped_refptr<VASurface> input_surface;
-  scoped_refptr<VASurface> reconstructed_surface;
-  if (native_input_mode_) {
-    if (!CreateSurfacesForGpuMemoryBufferEncoding(
-            *frame, encode_size, &input_surface, &reconstructed_surface)) {
-      return nullptr;
-    }
-  } else {
-    if (!CreateSurfacesForShmemEncoding(*frame, &input_surface,
-                                        &reconstructed_surface)) {
+  // iHD driver doesn't align a resolution for encoding properly. Align it only
+  // with encoder driver.
+  // TODO(https://github.com/intel/media-driver/issues/1232): Remove this
+  // workaround of aligning |encode_size|.
+  gfx::Size surface_size = encode_size;
+  if (!base::Contains(surface_usage_hints,
+                      VaapiWrapper::SurfaceUsageHint::kVideoProcessWrite)) {
+    surface_size = gfx::Size(base::bits::AlignUp(encode_size.width(), 16u),
+                             base::bits::AlignUp(encode_size.height(), 16u));
+  }
+
+  auto scoped_va_surfaces = vaapi_wrapper.CreateScopedVASurfaces(
+      kVaSurfaceFormat, surface_size, surface_usage_hints, num_surfaces,
+      /*visible_size=*/absl::nullopt,
+      /*va_fourcc=*/absl::nullopt);
+  if (scoped_va_surfaces.empty()) {
+    NOTIFY_ERROR(kPlatformFailureError, "Failed creating surfaces");
+    return false;
+  }
+
+  scoped_surfaces_map[encode_size] = std::move(scoped_va_surfaces);
+  return true;
+}
+
+scoped_refptr<VaapiWrapper>
+VaapiVideoEncodeAccelerator::CreateVppVaapiWrapper() {
+  DCHECK(!vpp_vaapi_wrapper_);
+  auto vpp_vaapi_wrapper = VaapiWrapper::Create(
+      VaapiWrapper::kVideoProcess, VAProfileNone,
+      EncryptionScheme::kUnencrypted,
+      base::BindRepeating(&ReportVaapiErrorToUMA,
+                          "Media.VaapiVideoEncodeAccelerator.Vpp.VAAPIError"));
+  if (!vpp_vaapi_wrapper) {
+    NOTIFY_ERROR(kPlatformFailureError, "Failed to initialize VppVaapiWrapper");
+    return nullptr;
+  }
+  // VA context for VPP is not associated with a specific resolution.
+  if (!vpp_vaapi_wrapper->CreateContext(gfx::Size())) {
+    NOTIFY_ERROR(kPlatformFailureError, "Failed creating Context for VPP");
+    return nullptr;
+  }
+
+  return vpp_vaapi_wrapper;
+}
+
+scoped_refptr<VASurface> VaapiVideoEncodeAccelerator::ExecuteBlitSurface(
+    const VASurface& source_surface,
+    const gfx::Rect source_visible_rect,
+    const gfx::Size& encode_size) {
+  if (!vpp_vaapi_wrapper_) {
+    vpp_vaapi_wrapper_ = CreateVppVaapiWrapper();
+    if (!vpp_vaapi_wrapper_) {
+      NOTIFY_ERROR(kPlatformFailureError, "Failed to create Vpp");
       return nullptr;
     }
   }
+
+  if (!CreateSurfacesIfNeeded(
+          *vpp_vaapi_wrapper_, available_vpp_dest_surfaces_, encode_size,
+          {VaapiWrapper::SurfaceUsageHint::kVideoProcessWrite,
+           VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
+          num_frames_in_flight_ + 1)) {
+    VLOGF(1) << "Failed to create or reuse " << (num_frames_in_flight_ + 1)
+             << " VASurfaces of size " << encode_size.ToString();
+    return nullptr;
+  }
+  if (available_vpp_dest_surfaces_[encode_size].empty()) {
+    DVLOGF(4) << "Not enough vpp destination surface available";
+    return nullptr;
+  }
+
+  auto blit_surface = GetAvailableVASurfaceAsRefCounted(
+      &available_vpp_dest_surfaces_[encode_size]);
+  DCHECK(blit_surface);
+  DCHECK(vpp_vaapi_wrapper_);
+  if (!vpp_vaapi_wrapper_->BlitSurface(source_surface, *blit_surface,
+                                       source_visible_rect,
+                                       gfx::Rect(encode_size))) {
+    NOTIFY_ERROR(kPlatformFailureError,
+                 "Failed BlitSurface on frame size: "
+                     << source_surface.size().ToString()
+                     << " (visible rect: " << source_visible_rect.ToString()
+                     << ") -> encode size: " << encode_size.ToString());
+    return nullptr;
+  }
+
+  return blit_surface;
+}
+
+std::unique_ptr<VaapiVideoEncoderDelegate::EncodeJob>
+VaapiVideoEncodeAccelerator::CreateEncodeJob(
+    scoped_refptr<VideoFrame> frame,
+    bool force_keyframe,
+    scoped_refptr<VASurface> input_surface,
+    scoped_refptr<VASurface> reconstructed_surface) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
+  DCHECK(frame);
   DCHECK(input_surface && reconstructed_surface);
 
   auto coded_buffer = vaapi_wrapper_->CreateVABuffer(VAEncCodedBufferType,
@@ -794,16 +818,37 @@ void VaapiVideoEncodeAccelerator::EncodePendingInputs() {
       continue;
     }
 
+    const size_t num_spatial_layers = spatial_layer_resolutions.size();
+    std::vector<scoped_refptr<VASurface>> input_surfaces;
+    std::vector<scoped_refptr<VASurface>> reconstructed_surfaces;
+    if (native_input_mode_) {
+      if (!CreateSurfacesForGpuMemoryBufferEncoding(
+              *input_frame->frame, spatial_layer_resolutions, &input_surfaces,
+              &reconstructed_surfaces)) {
+        return;
+      }
+    } else {
+      DCHECK_EQ(num_spatial_layers, 1u);
+      input_surfaces.resize(1u);
+      reconstructed_surfaces.resize(1u);
+      if (!CreateSurfacesForShmemEncoding(*input_frame->frame,
+                                          &input_surfaces[0],
+                                          &reconstructed_surfaces[0])) {
+        return;
+      }
+    }
+
     // Encoding different spatial layers for |input_frame|.
     std::vector<std::unique_ptr<EncodeJob>> jobs;
-    for (size_t spatial_idx = 0; spatial_idx < spatial_layer_resolutions.size();
+    for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
          ++spatial_idx) {
       std::unique_ptr<EncodeJob> job;
       TRACE_EVENT0("media,gpu", "VAVEA::FromCreateEncodeJobToReturn");
       const bool force_key =
           (spatial_idx == 0 ? input_frame->force_keyframe : false);
       job = CreateEncodeJob(input_frame->frame, force_key,
-                            spatial_layer_resolutions[spatial_idx]);
+                            std::move(input_surfaces[spatial_idx]),
+                            std::move(reconstructed_surfaces[spatial_idx]));
       if (!job)
         return;
 
@@ -966,13 +1011,13 @@ void VaapiVideoEncodeAccelerator::DestroyTask() {
   if (vaapi_wrapper_)
     vaapi_wrapper_->DestroyContext();
 
-  available_va_surfaces_.clear();
+  available_encode_surfaces_.clear();
   available_va_buffer_ids_.clear();
 
   if (vpp_vaapi_wrapper_)
     vpp_vaapi_wrapper_->DestroyContext();
 
-  available_vpp_va_surfaces_.clear();
+  available_vpp_dest_surfaces_.clear();
 
   while (!available_bitstream_buffers_.empty())
     available_bitstream_buffers_.pop();

media/gpu/vaapi/vaapi_video_encode_accelerator.h

@@ -20,10 +20,10 @@
 #include "media/gpu/media_gpu_export.h"
 #include "media/gpu/vaapi/vaapi_utils.h"
 #include "media/gpu/vaapi/vaapi_video_encoder_delegate.h"
+#include "media/gpu/vaapi/vaapi_wrapper.h"
 #include "media/video/video_encode_accelerator.h"
 
 namespace media {
-class VaapiWrapper;
 
 // A VideoEncodeAccelerator implementation that uses VA-API
 // (https://01.org/vaapi) for HW-accelerated video encode.
@@ -67,16 +67,17 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
     }
   };
 
+  // TODO(crbug.com/1186051): Use base::small_map.
+  using ScopedVASurfacesMap =
+      std::map<gfx::Size,
+               std::vector<std::unique_ptr<ScopedVASurface>>,
+               SizeComparator>;
+
   // Holds input frames coming from the client ready to be encoded.
   struct InputFrameRef;
   // Holds output buffers coming from the client ready to be filled.
   struct BitstreamBufferRef;
 
-  // one surface for input data.
-  // one surface for reconstructed picture, which is later used for reference.
-  static constexpr size_t kNumSurfacesPerInputVideoFrame = 1;
-  static constexpr size_t kNumSurfacesForOutputPicture = 1;
-
   //
   // Tasks for each of the VEA interface calls to be executed on
   // |encoder_task_runner_|.
@@ -99,23 +100,17 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   void DestroyTask();
   void FlushTask(FlushCallback flush_callback);
 
-  // Blits |input_surface| to an internally-allocated |input_visible_rect|
-  // surface, returning it. If |vpp_vaapi_wrapper_| is empty, this will create
-  // it and corresponding surfaces. Returns nullptr on failure.
-  scoped_refptr<VASurface> BlitSurfaceWithCreateVppIfNeeded(
-      const VASurface& input_surface,
-      const gfx::Rect& input_visible_rect,
-      const gfx::Size& encode_size,
-      size_t num_va_surfaces);
-
   // Create input and reconstructed surfaces used in encoding whose sizes are
-  // |encode_size| from GpuMemoryBuffer-based VideoFrame |frame|. This must be
-  // called only in native input mode.
+  // |spatial_layer_resolutions| from GpuMemoryBuffer-based VideoFrame |frame|.
+  // The created surfaces for input to an encoder driver are filled into
+  // |input_surfaces| and, ones used as reconstructed surfaces by the driver are
+  // filled to |reconstructed_surfaces|. This must be called only in native
+  // input mode.
   bool CreateSurfacesForGpuMemoryBufferEncoding(
       const VideoFrame& frame,
-      const gfx::Size& encode_size,
-      scoped_refptr<VASurface>* input_surface,
-      scoped_refptr<VASurface>* reconstructed_surface);
+      const std::vector<gfx::Size>& spatial_layer_resolutions,
+      std::vector<scoped_refptr<VASurface>>* input_surfaces,
+      std::vector<scoped_refptr<VASurface>>* reconstructed_surfaces);
 
   // Create input and reconstructed surfaces used in encoding from SharedMemory
   // VideoFrame |frame|. This must be called only in non native input mode.
@@ -124,12 +119,35 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
       scoped_refptr<VASurface>* input_surface,
       scoped_refptr<VASurface>* reconstructed_surface);
 
+  // Creates |num_surfaces| ScopedVASurfaces using |vaapi_wrapper| whose sizes
+  // are |encode_size| with |surface_usage_hints|. Returns false if the surfaces
+  // fail to be created successfully.
+  // The created surfaces are filled into |scoped_surfaces_map[encode_size]|.
+  bool CreateSurfacesIfNeeded(
+      VaapiWrapper& vaapi_wrapper,
+      ScopedVASurfacesMap& scoped_surfaces_map,
+      const gfx::Size& encode_size,
+      const std::vector<VaapiWrapper::SurfaceUsageHint>& surface_usage_hints,
+      size_t num_surfaces);
+
+  // Creates |vpp_vaapi_wrapper_| if it hasn't been created.
+  scoped_refptr<VaapiWrapper> CreateVppVaapiWrapper();
+  // Executes BlitSurface() using |vpp_vaapi_wrapper_| with |source_surface|,
+  // |source_visible_rect|. Returns the destination VASurface in BlitSurface()
+  // whose size is |encode_size| on success, otherwise nullptr.
+  scoped_refptr<VASurface> ExecuteBlitSurface(
+      const VASurface& source_surface,
+      const gfx::Rect source_visible_rect,
+      const gfx::Size& encode_size);
+
   // Checks if sufficient resources for a new encode job with |frame| as input
   // are available, and if so, claims them by associating them with
   // a EncodeJob, and returns the newly-created job, nullptr otherwise.
-  std::unique_ptr<EncodeJob> CreateEncodeJob(scoped_refptr<VideoFrame> frame,
-                                             bool force_keyframe,
-                                             const gfx::Size& encode_size);
+  std::unique_ptr<EncodeJob> CreateEncodeJob(
+      scoped_refptr<VideoFrame> frame,
+      bool force_keyframe,
+      scoped_refptr<VASurface> input_surface,
+      scoped_refptr<VASurface> reconstructed_surface);
 
   // Continues encoding frames as long as input_queue_ is not empty, and we are
   // able to create new EncodeJobs.
@@ -203,14 +221,6 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   // input.
   bool native_input_mode_ = false;
 
-  // The number of va surfaces required for one video frame on Encode().
-  // In |native_input_mode_|, one surface for input data is created from DmaBufs
-  // of incoming VideoFrame. One surface for reconstructed picture is always
-  // needed, which is later used for reference.
-  // Therefore, |va_surfaces_per_video_frame| is one in |native_input_mode_|,
-  // and two otherwise.
-  size_t va_surfaces_per_video_frame_;
-
   // The number of frames that needs to be held on encoding.
   size_t num_frames_in_flight_;
 
@@ -224,14 +234,12 @@ class MEDIA_GPU_EXPORT VaapiVideoEncodeAccelerator
   // Should only be used on |encoder_task_runner_|.
   std::unique_ptr<VaapiVideoEncoderDelegate> encoder_;
 
-  // VA surfaces available for encoding.
-  std::vector<std::unique_ptr<ScopedVASurface>> available_va_surfaces_;
-  // VA surfaces available for scaling.
-  // TODO(crbug.com/1186051): Use base::small_map.
-  std::map<gfx::Size,
-           std::vector<std::unique_ptr<ScopedVASurface>>,
-           SizeComparator>
-      available_vpp_va_surfaces_;
+  // Map of available input or reconstructed surfaces for encoding indexed by a
+  // layer resolution.
+  ScopedVASurfacesMap available_encode_surfaces_;
+  // Map of available destination surfaces for scaling and cropping, and input
+  // surfaces for encoding indexed by a layer resolution..
+  ScopedVASurfacesMap available_vpp_dest_surfaces_;
 
   // VA buffers for coded frames.
   std::vector<VABufferID> available_va_buffer_ids_;

media/gpu/vaapi/vaapi_video_encode_accelerator_unittest.cc

@@ -8,6 +8,7 @@
 #include <numeric>
 #include <vector>
 
+#include "base/bits.h"
 #include "base/run_loop.h"
 #include "base/test/gmock_callback_support.h"
 #include "base/test/task_environment.h"
@@ -149,13 +150,7 @@ class MockVaapiWrapper : public VaapiWrapper {
   explicit MockVaapiWrapper(CodecMode mode) : VaapiWrapper(mode) {}
 
   MOCK_METHOD2(GetVAEncMaxNumOfRefFrames, bool(VideoCodecProfile, size_t*));
-  MOCK_METHOD5(CreateContextAndScopedVASurfaces,
-               std::vector<std::unique_ptr<ScopedVASurface>>(
-                   unsigned int,
-                   const gfx::Size&,
-                   const std::vector<SurfaceUsageHint>&,
-                   size_t,
-                   const absl::optional<gfx::Size>&));
+  MOCK_METHOD1(CreateContext, bool(const gfx::Size&));
   MOCK_METHOD6(CreateScopedVASurfaces,
                std::vector<std::unique_ptr<ScopedVASurface>>(
                    unsigned int,
@@ -288,37 +283,20 @@ class VaapiVideoEncodeAcceleratorTest
     const size_t num_spatial_layers = config.spatial_layers.size();
     // Scaling is needed only for non highest spatial layer, so here the vpp
     // number is |num_spatial_layers - 1|.
-    vpp_svc_va_surface_ids_.resize(num_spatial_layers - 1);
-    vpp_svc_mock_vaapi_wrapper_ =
+    va_encode_surface_ids_.resize(num_spatial_layers);
+    va_vpp_dest_surface_ids_.resize(num_spatial_layers - 1);
+    mock_vpp_vaapi_wrapper_ =
         base::MakeRefCounted<MockVaapiWrapper>(VaapiWrapper::kVideoProcess);
     auto* vaapi_encoder =
         reinterpret_cast<VaapiVideoEncodeAccelerator*>(encoder_.get());
-    vaapi_encoder->vpp_vaapi_wrapper_ = vpp_svc_mock_vaapi_wrapper_;
+    vaapi_encoder->vpp_vaapi_wrapper_ = mock_vpp_vaapi_wrapper_;
 
     EXPECT_CALL(*mock_encoder_,
                 Initialize(_, MatchesVaapiVideoEncoderDelegateConfig(
                                   kMaxNumOfRefFrames, kBitrateControl)))
         .WillOnce(Return(true));
-    EXPECT_CALL(*mock_vaapi_wrapper_,
-                CreateContextAndScopedVASurfaces(
-                    VA_RT_FORMAT_YUV420, kDefaultEncodeSize,
-                    std::vector<VaapiWrapper::SurfaceUsageHint>{
-                        VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
-                    _, _))
-        .WillOnce(
-            WithArgs<0, 1, 3>([&surface_ids = this->va_surface_ids_,
-                               &vaapi_wrapper = this->mock_vaapi_wrapper_](
-                                  unsigned int format, const gfx::Size& size,
-                                  size_t num_surfaces) {
-              surface_ids.resize(num_surfaces);
-              std::iota(surface_ids.begin(), surface_ids.end(), 1);
-              std::vector<std::unique_ptr<ScopedVASurface>> va_surfaces;
-              for (const VASurfaceID id : surface_ids) {
-                va_surfaces.push_back(std::make_unique<ScopedVASurface>(
-                    vaapi_wrapper, id, size, format));
-              }
-              return va_surfaces;
-            }));
+    EXPECT_CALL(*mock_vaapi_wrapper_, CreateContext(kDefaultEncodeSize))
+        .WillOnce(Return(true));
     EXPECT_CALL(client_, RequireBitstreamBuffers(_, kDefaultEncodeSize, _))
         .WillOnce(WithArgs<2>([this](size_t output_buffer_size) {
           this->output_buffer_size_ = output_buffer_size;
@@ -335,6 +313,30 @@ class VaapiVideoEncodeAcceleratorTest
       bool use_temporal_layer_encoding) {
     ::testing::InSequence s;
 
+    constexpr VASurfaceID kInputSurfaceId = 1234;
+    EXPECT_CALL(*mock_vaapi_wrapper_,
+                CreateScopedVASurfaces(
+                    VA_RT_FORMAT_YUV420, kDefaultEncodeSize,
+                    std::vector<VaapiWrapper::SurfaceUsageHint>{
+                        VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
+                    _, absl::optional<gfx::Size>(), absl::optional<uint32_t>()))
+        .WillOnce(
+            WithArgs<0, 1, 3>([&surface_ids = this->va_encode_surface_ids_[0],
+                               &vaapi_wrapper = this->mock_vaapi_wrapper_,
+                               va_surface_id = kInputSurfaceId](
+                                  unsigned int format, const gfx::Size& size,
+                                  size_t num_surfaces) {
+              surface_ids.resize(num_surfaces);
+              std::iota(surface_ids.begin(), surface_ids.end(), 1);
+              surface_ids.back() = va_surface_id;
+              std::vector<std::unique_ptr<ScopedVASurface>> va_surfaces;
+              for (const VASurfaceID id : surface_ids) {
+                va_surfaces.push_back(std::make_unique<ScopedVASurface>(
+                    vaapi_wrapper, id, size, format));
+              }
+              return va_surfaces;
+            }));
+
     constexpr VABufferID kCodedBufferId = 123;
     EXPECT_CALL(*mock_vaapi_wrapper_,
                 CreateVABuffer(VAEncCodedBufferType, output_buffer_size_))
@@ -342,29 +344,27 @@ class VaapiVideoEncodeAcceleratorTest
           return ScopedVABuffer::CreateForTesting(
               kCodedBufferId, VAEncCodedBufferType, buffer_size);
         }));
-    ASSERT_FALSE(va_surface_ids_.empty());
-    const VASurfaceID kInputSurfaceId = va_surface_ids_.back();
+
     EXPECT_CALL(*mock_encoder_, PrepareEncodeJob(_))
-        .WillOnce(WithArgs<0>(
-            [encoder = encoder_.get(), kCodedBufferId,
-             use_temporal_layer_encoding,
-             kInputSurfaceId](VaapiVideoEncoderDelegate::EncodeJob* job) {
-              if (use_temporal_layer_encoding) {
-                // Set Vp9Metadata on temporal layer encoding.
-                CodecPicture* picture = job->picture().get();
-                reinterpret_cast<VP9Picture*>(picture)->metadata_for_encoding =
-                    Vp9Metadata();
-              }
-              auto* vaapi_encoder =
-                  reinterpret_cast<VaapiVideoEncodeAccelerator*>(encoder);
-              job->AddPostExecuteCallback(base::BindOnce(
-                  &VP9VaapiVideoEncoderDelegate::NotifyEncodedChunkSize,
-                  base::Unretained(
-                      reinterpret_cast<VP9VaapiVideoEncoderDelegate*>(
-                          vaapi_encoder->encoder_.get())),
-                  kCodedBufferId, kInputSurfaceId));
-              return true;
-            }));
+        .WillOnce(WithArgs<0>([encoder = encoder_.get(), kCodedBufferId,
+                               use_temporal_layer_encoding,
+                               va_surface_id = kInputSurfaceId](
+                                  VaapiVideoEncoderDelegate::EncodeJob* job) {
+          if (use_temporal_layer_encoding) {
+            // Set Vp9Metadata on temporal layer encoding.
+            CodecPicture* picture = job->picture().get();
+            reinterpret_cast<VP9Picture*>(picture)->metadata_for_encoding =
+                Vp9Metadata();
+          }
+          auto* vaapi_encoder =
+              reinterpret_cast<VaapiVideoEncodeAccelerator*>(encoder);
+          job->AddPostExecuteCallback(base::BindOnce(
+              &VP9VaapiVideoEncoderDelegate::NotifyEncodedChunkSize,
+              base::Unretained(reinterpret_cast<VP9VaapiVideoEncoderDelegate*>(
+                  vaapi_encoder->encoder_.get())),
+              kCodedBufferId, va_surface_id));
+          return true;
+        }));
     EXPECT_CALL(
         *mock_vaapi_wrapper_,
         UploadVideoFrameToSurface(_, kInputSurfaceId, kDefaultEncodeSize))
@@ -456,36 +456,38 @@ class VaapiVideoEncodeAcceleratorTest
 
     ::testing::InSequence s;
 
+    // Create VASurface from GpuMemory-based VideoFrame.
+    const VASurfaceID kSourceSurfaceId = 123456;
+    EXPECT_CALL(*mock_vaapi_wrapper_, CreateVASurfaceForPixmap(_, _))
+        .WillOnce(
+            Return(new VASurface(kSourceSurfaceId, kDefaultEncodeSize,
+                                 VA_RT_FORMAT_YUV420, base::DoNothing())));
+
+    constexpr VASurfaceID kVppDestSurfaceIds[] = {456, 457};
+    constexpr VASurfaceID kEncodeSurfaceIds[] = {458, 459, 460};
     std::vector<gfx::Size> svc_resolutions =
         GetDefaultSVCResolutions(num_spatial_layers);
-    constexpr VABufferID kCodedBufferIds[] = {123, 124, 125};
+    // Create Surfaces.
     for (size_t i = 0; i < num_spatial_layers; ++i) {
-      const VASurfaceID kInputSurfaceId = va_surface_ids_.back();
-      const gfx::Size layer_size = svc_resolutions[i];
-      EXPECT_CALL(*mock_vaapi_wrapper_, CreateVASurfaceForPixmap(_, _))
-          .WillOnce(WithArgs<0>([kInputSurfaceId, layer_size]() {
-            return new VASurface(kInputSurfaceId, layer_size,
-                                 VA_RT_FORMAT_YUV420, base::DoNothing());
-          }));
-
-      // Scaling and vpp only needed for non highest spatial layer.
       if (i < num_spatial_layers - 1) {
-        if (vpp_svc_va_surface_ids_[i].empty()) {
+        if (va_vpp_dest_surface_ids_[i].empty()) {
           EXPECT_CALL(
-              *vpp_svc_mock_vaapi_wrapper_,
+              *mock_vpp_vaapi_wrapper_,
               CreateScopedVASurfaces(
-                  VA_RT_FORMAT_YUV420, layer_size,
+                  VA_RT_FORMAT_YUV420, svc_resolutions[i],
                   std::vector<VaapiWrapper::SurfaceUsageHint>{
                       VaapiWrapper::SurfaceUsageHint::kVideoProcessWrite,
                       VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
                   _, absl::optional<gfx::Size>(), absl::optional<uint32_t>()))
               .WillOnce(WithArgs<0, 1, 3>(
-                  [&surface_ids = this->vpp_svc_va_surface_ids_[i],
-                   &vaapi_wrapper = this->vpp_svc_mock_vaapi_wrapper_](
+                  [&surface_ids = this->va_vpp_dest_surface_ids_[i],
+                   &vaapi_wrapper = this->mock_vpp_vaapi_wrapper_,
+                   vpp_dest_surface_id = kVppDestSurfaceIds[i]](
                       unsigned int format, const gfx::Size& size,
                       size_t num_surfaces) {
                     surface_ids.resize(num_surfaces);
                     std::iota(surface_ids.begin(), surface_ids.end(), 1);
+                    surface_ids.back() = vpp_dest_surface_id;
                     std::vector<std::unique_ptr<ScopedVASurface>> va_surfaces;
                     for (const VASurfaceID id : surface_ids) {
                       va_surfaces.push_back(std::make_unique<ScopedVASurface>(
@@ -494,15 +496,54 @@ class VaapiVideoEncodeAcceleratorTest
                     return va_surfaces;
                   }));
         }
-
         absl::optional<gfx::Rect> default_rect = gfx::Rect(kDefaultEncodeSize);
-        absl::optional<gfx::Rect> layer_rect = gfx::Rect(layer_size);
-        EXPECT_CALL(*vpp_svc_mock_vaapi_wrapper_,
+        absl::optional<gfx::Rect> layer_rect = gfx::Rect(svc_resolutions[i]);
+        EXPECT_CALL(*mock_vpp_vaapi_wrapper_,
                     BlitSurface(_, _, default_rect, layer_rect,
                                 VideoRotation::VIDEO_ROTATION_0))
             .WillOnce(Return(true));
       }
 
+      // For reconstructed surface.
+      if (va_encode_surface_ids_[i].empty()) {
+        // TODO(https://github.com/intel/media-driver/issues/1232): Remove this
+        // workaround of aligning |encode_size|.
+        gfx::Size aligned_size(
+            base::bits::AlignUp(svc_resolutions[i].width(), 16),
+            base::bits::AlignUp(svc_resolutions[i].height(), 16));
+
+        EXPECT_CALL(
+            *mock_vaapi_wrapper_,
+            CreateScopedVASurfaces(
+                VA_RT_FORMAT_YUV420, aligned_size,
+                std::vector<VaapiWrapper::SurfaceUsageHint>{
+                    VaapiWrapper::SurfaceUsageHint::kVideoEncoder},
+                _, absl::optional<gfx::Size>(), absl::optional<uint32_t>()))
+            .WillOnce(WithArgs<0, 1, 3>(
+                [&surface_ids = this->va_encode_surface_ids_[i],
+                 &vaapi_wrapper = this->mock_vaapi_wrapper_,
+                 va_encode_surface_id = kEncodeSurfaceIds[i]](
+                    unsigned int format, const gfx::Size& size,
+                    size_t num_surfaces) {
+                  surface_ids.resize(num_surfaces);
+                  std::iota(surface_ids.begin(), surface_ids.end(), 1);
+                  surface_ids.back() = va_encode_surface_id;
+                  std::vector<std::unique_ptr<ScopedVASurface>> va_surfaces;
+                  for (const VASurfaceID id : surface_ids) {
+                    va_surfaces.push_back(std::make_unique<ScopedVASurface>(
+                        vaapi_wrapper, id, size, format));
+                  }
+                  return va_surfaces;
+                }));
+      }
+    }
+
+    for (size_t i = 0; i < num_spatial_layers; ++i) {
+    }
+
+    // Create CodedBuffers in creating EncodeJobs.
+    constexpr VABufferID kCodedBufferIds[] = {123, 124, 125};
+    for (size_t i = 0; i < num_spatial_layers; ++i) {
       const VABufferID kCodedBufferId = kCodedBufferIds[i];
       EXPECT_CALL(*mock_vaapi_wrapper_,
                   CreateVABuffer(VAEncCodedBufferType, output_buffer_size_))
@@ -514,13 +555,13 @@ class VaapiVideoEncodeAcceleratorTest
 
     for (size_t i = 0; i < num_spatial_layers; ++i) {
       const VABufferID kCodedBufferId = kCodedBufferIds[i];
-      std::vector<VASurfaceID>* surfaces = i < num_spatial_layers - 1
-                                               ? &vpp_svc_va_surface_ids_[i]
-                                               : &va_surface_ids_;
+      const VASurfaceID input_surface_id =
+          i < num_spatial_layers - 1 ? kVppDestSurfaceIds[i] : kSourceSurfaceId;
+
       EXPECT_CALL(*mock_encoder_, PrepareEncodeJob(_))
-          .WillOnce(
-              WithArgs<0>([encoder = encoder_.get(), kCodedBufferId, surfaces](
-                              VaapiVideoEncoderDelegate::EncodeJob* job) {
+          .WillOnce(WithArgs<0>(
+              [encoder = encoder_.get(), kCodedBufferId,
+               input_surface_id](VaapiVideoEncoderDelegate::EncodeJob* job) {
                 // Set Vp9Metadata on spatial layer encoding.
                 CodecPicture* picture = job->picture().get();
                 reinterpret_cast<VP9Picture*>(picture)->metadata_for_encoding =
@@ -532,7 +573,7 @@ class VaapiVideoEncodeAcceleratorTest
                     base::Unretained(
                         reinterpret_cast<VP9VaapiVideoEncoderDelegate*>(
                             vaapi_encoder->encoder_.get())),
-                    kCodedBufferId, surfaces->back()));
+                    kCodedBufferId, input_surface_id));
                 return true;
               }));
       EXPECT_CALL(*mock_vaapi_wrapper_, ExecuteAndDestroyPendingBuffers(_))
@@ -580,15 +621,15 @@ class VaapiVideoEncodeAcceleratorTest
   }
 
   size_t output_buffer_size_ = 0;
-  std::vector<VASurfaceID> va_surface_ids_;
-  std::vector<std::vector<VASurfaceID>> vpp_svc_va_surface_ids_;
+  std::vector<std::vector<VASurfaceID>> va_encode_surface_ids_;
+  std::vector<std::vector<VASurfaceID>> va_vpp_dest_surface_ids_;
   base::test::TaskEnvironment task_environment_;
   MockVideoEncodeAcceleratorClient client_;
   // |encoder_| is a VideoEncodeAccelerator to use its specialized Deleter that
   // calls Destroy() so that destruction threading is respected.
   std::unique_ptr<VideoEncodeAccelerator> encoder_;
   scoped_refptr<MockVaapiWrapper> mock_vaapi_wrapper_;
-  scoped_refptr<MockVaapiWrapper> vpp_svc_mock_vaapi_wrapper_;
+  scoped_refptr<MockVaapiWrapper> mock_vpp_vaapi_wrapper_;
   MockVP9VaapiVideoEncoderDelegate* mock_encoder_ = nullptr;
 };