media/gpu/vaapi: Enable to drop frame in VP8 encoding

This CL adds a drop frame support to VA-API vp8 encoding.

Design doc: go/frame-drop-VEA

Bug: b:280363228
Test: media_unittests
Test: video_encode_accelerator_tests

Change-Id: I8c7a685500919ca21b31b3f83ea0cb1ed763b04c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5052467
Reviewed-by: Justin Green <greenjustin@google.com>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1237946}

media/gpu/vaapi/vaapi_video_encode_accelerator.cc

@@ -467,27 +467,32 @@ void VaapiVideoEncodeAccelerator::ReturnBitstreamBuffer(
     const EncodeResult& encode_result,
     const BitstreamBuffer& buffer) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(encoder_sequence_checker_);
-  const base::UnsafeSharedMemoryRegion& shm_region = buffer.region();
-  DCHECK(shm_region.IsValid());
-  base::WritableSharedMemoryMapping shm_mapping = shm_region.Map();
-  uint8_t* target_data = shm_mapping.GetMemoryAs<uint8_t>();
-  size_t data_size = 0;
-  // vaSyncSurface() is not necessary because GetEncodedChunkSize() has been
-  // called in VaapiVideoEncoderDelegate::Encode().
-  if (!vaapi_wrapper_->DownloadFromVABuffer(
-          encode_result.coded_buffer_id(), /*sync_surface_id=*/absl::nullopt,
-          target_data, shm_mapping.size(), &data_size)) {
-    NotifyError({EncoderStatus::Codes::kEncoderHardwareDriverError,
-                 "Failed downloading coded buffer"});
-    return;
-  }
 
   auto metadata = encode_result.metadata();
-  DCHECK_NE(metadata.payload_size_bytes, 0u);
 
-  DVLOGF(4) << "Returning bitstream buffer "
-            << (metadata.key_frame ? "(keyframe)" : "")
-            << " id: " << buffer.id() << " size: " << data_size;
+  if (!encode_result.IsFrameDropped()) {
+    const base::UnsafeSharedMemoryRegion& shm_region = buffer.region();
+    DCHECK(shm_region.IsValid());
+    base::WritableSharedMemoryMapping shm_mapping = shm_region.Map();
+    uint8_t* target_data = shm_mapping.GetMemoryAs<uint8_t>();
+    size_t data_size = 0;
+    // vaSyncSurface() is not necessary because GetEncodedChunkSize() has been
+    // called in VaapiVideoEncoderDelegate::Encode().
+    if (!vaapi_wrapper_->DownloadFromVABuffer(
+            encode_result.coded_buffer_id(), /*sync_surface_id=*/absl::nullopt,
+            target_data, shm_mapping.size(), &data_size)) {
+      NotifyError({EncoderStatus::Codes::kEncoderHardwareDriverError,
+                   "Failed downloading coded buffer"});
+      return;
+    }
+    CHECK_EQ(metadata.payload_size_bytes, data_size);
+    DVLOGF(4) << "Returning bitstream buffer "
+              << (metadata.key_frame ? "(keyframe)" : "")
+              << " id: " << buffer.id() << " size: " << data_size;
+  } else {
+    CHECK_EQ(metadata.payload_size_bytes, 0u);
+    DVLOGF(4) << "Drop frame bitstream_buffer_id=" << buffer.id();
+  }
 
   TRACE_EVENT2("media,gpu", "VAVEA::BitstreamBufferReady", "timestamp",
                metadata.timestamp.InMicroseconds(), "bitstream_buffer_id",

media/gpu/vaapi/vaapi_video_encoder_delegate.cc

@@ -55,6 +55,7 @@ base::TimeDelta VaapiVideoEncoderDelegate::EncodeJob::timestamp() const {
 }
 
 VABufferID VaapiVideoEncoderDelegate::EncodeJob::coded_buffer_id() const {
+  CHECK(coded_buffer_);
   return coded_buffer_->id();
 }
 
@@ -80,6 +81,7 @@ VaapiVideoEncoderDelegate::EncodeResult&
 VaapiVideoEncoderDelegate::EncodeResult::operator=(EncodeResult&&) = default;
 
 VABufferID VaapiVideoEncoderDelegate::EncodeResult::coded_buffer_id() const {
+  CHECK(coded_buffer_);
   return coded_buffer_->id();
 }
 
@@ -125,6 +127,14 @@ bool VaapiVideoEncoderDelegate::Encode(EncodeJob& encode_job) {
     return false;
   }
 
+  if (result == PrepareEncodeJobResult::kDrop) {
+    // An encoder must not drop a keyframe.
+    CHECK(!encode_job.IsKeyframeRequested());
+    DVLOGF(3) << "Drop frame";
+    encode_job.DropFrame();
+    return true;
+  }
+
   if (!vaapi_wrapper_->ExecuteAndDestroyPendingBuffers(
           encode_job.input_surface_id())) {
     VLOGF(1) << "Failed to execute encode";
@@ -138,6 +148,11 @@ absl::optional<VaapiVideoEncoderDelegate::EncodeResult>
 VaapiVideoEncoderDelegate::GetEncodeResult(
     std::unique_ptr<EncodeJob> encode_job) {
   TRACE_EVENT0("media,gpu", "VAVEDelegate::GetEncodeResult");
+  if (encode_job->IsFrameDropped()) {
+    return absl::make_optional<EncodeResult>(nullptr,
+                                             GetMetadata(*encode_job, 0u));
+  }
+
   const VASurfaceID va_surface_id = encode_job->input_surface_id();
   const uint64_t encoded_chunk_size = vaapi_wrapper_->GetEncodedChunkSize(
       encode_job->coded_buffer_id(), va_surface_id);

media/gpu/vaapi/vaapi_video_encoder_delegate.h

@@ -62,6 +62,7 @@ class VaapiVideoEncoderDelegate {
 
     VABufferID coded_buffer_id() const;
     const BitstreamBufferMetadata& metadata() const;
+    bool IsFrameDropped() const { return !coded_buffer_; }
 
    private:
     std::unique_ptr<ScopedVABuffer> coded_buffer_;
@@ -107,6 +108,9 @@ class VaapiVideoEncoderDelegate {
     // Returns true if this job has been requested to produce a keyframe.
     bool IsKeyframeRequested() const { return keyframe_; }
 
+    void DropFrame() { coded_buffer_.reset(); }
+    bool IsFrameDropped() const { return !coded_buffer_; }
+
     base::TimeDelta timestamp() const;
 
     // VA-API specific methods.
@@ -178,8 +182,9 @@ class VaapiVideoEncoderDelegate {
   friend class VaapiVideoEncodeAcceleratorTest;
 
   enum class PrepareEncodeJobResult {
-    kSuccess,
-    kFail,
+    kSuccess,  // Submit the encode job successfully.
+    kFail,     // Error happens in submitting the encode job.
+    kDrop,     // Encode job is dropped. An returned encoded chunk is empty.
   };
 
   virtual BitstreamBufferMetadata GetMetadata(const EncodeJob& encode_job,

media/gpu/vaapi/vp8_vaapi_video_encoder_delegate.cc

@@ -113,6 +113,7 @@ libvpx::VP8RateControlRtcConfig CreateRateControlConfig(
     rc_cfg.ts_rate_decimator[tid] = 1u << (num_temporal_layers - tid - 1);
   }
 
+  rc_cfg.frame_drop_thresh = encode_params.drop_frame_thresh;
   return rc_cfg;
 }
 
@@ -254,7 +255,6 @@ bool VP8TLEncodingIsEnabled() {
 
 VP8VaapiVideoEncoderDelegate::EncodeParams::EncodeParams()
     : kf_period_frames(kKFPeriod),
-      framerate(0),
       min_qp(kMinQP),
       max_qp(kMaxQP) {}
 
@@ -330,6 +330,8 @@ bool VP8VaapiVideoEncoderDelegate::Initialize(
     current_params_.max_qp = kScreenMaxQP;
   }
 
+  current_params_.drop_frame_thresh = config.drop_frame_thresh_percentage;
+
   // |rate_ctrl_| might be injected for tests.
   if (!rate_ctrl_) {
     rate_ctrl_ = libvpx::VP8RateControlRTC::Create(CreateRateControlConfig(
@@ -379,7 +381,11 @@ VP8VaapiVideoEncoderDelegate::PrepareEncodeJob(EncodeJob& encode_job) {
   // We only use |last_frame| for a reference frame. This follows the behavior
   // of libvpx encoder in chromium webrtc use case.
   std::array<bool, kNumVp8ReferenceBuffers> ref_frames_used;
-  SetFrameHeader(frame_num_, *picture, ref_frames_used);
+  if (auto result = SetFrameHeader(frame_num_, *picture, ref_frames_used);
+      result != PrepareEncodeJobResult::kSuccess) {
+    return result;
+  }
+
   DCHECK(!picture->frame_hdr->IsKeyframe() ||
          !base::Contains(ref_frames_used, true));
 
@@ -400,7 +406,6 @@ BitstreamBufferMetadata VP8VaapiVideoEncoderDelegate::GetMetadata(
     const EncodeJob& encode_job,
     size_t payload_size) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
-
   auto metadata =
       VaapiVideoEncoderDelegate::GetMetadata(encode_job, payload_size);
   auto picture = GetVP8Picture(encode_job);
@@ -423,7 +428,7 @@ void VP8VaapiVideoEncoderDelegate::BitrateControlUpdate(
   DVLOGF(4) << "temporal_idx="
             << (metadata.vp8 ? metadata.vp8->temporal_idx : 0)
             << ", encoded chunk size=" << metadata.payload_size_bytes;
-
+  CHECK_NE(metadata.payload_size_bytes, 0u);
   rate_ctrl_->PostEncodeUpdate(metadata.payload_size_bytes);
 }
 
@@ -475,7 +480,8 @@ bool VP8VaapiVideoEncoderDelegate::UpdateRates(
   return true;
 }
 
-void VP8VaapiVideoEncoderDelegate::SetFrameHeader(
+VaapiVideoEncoderDelegate::PrepareEncodeJobResult
+VP8VaapiVideoEncoderDelegate::SetFrameHeader(
     size_t frame_num,
     VP8Picture& picture,
     std::array<bool, kNumVp8ReferenceBuffers>& ref_frames_used) {
@@ -516,7 +522,11 @@ void VP8VaapiVideoEncoderDelegate::SetFrameHeader(
           ? picture.metadata_for_encoding->temporal_idx
           : 0;
 
-  rate_ctrl_->ComputeQP(frame_params);
+  if (rate_ctrl_->ComputeQP(frame_params) == libvpx::FrameDropDecision::kDrop) {
+    CHECK(!keyframe);
+    DVLOGF(3) << "Drop frame";
+    return PrepareEncodeJobResult::kDrop;
+  }
   picture.frame_hdr->quantization_hdr.y_ac_qi = rate_ctrl_->GetQP();
   picture.frame_hdr->loopfilter_hdr.level =
       base::checked_cast<uint8_t>(rate_ctrl_->GetLoopfilterLevel());
@@ -529,6 +539,7 @@ void VP8VaapiVideoEncoderDelegate::SetFrameHeader(
                     ? " temporal id=" +
                           base::NumberToString(frame_params.temporal_layer_id)
                     : "");
+  return PrepareEncodeJobResult::kSuccess;
 }
 
 void VP8VaapiVideoEncoderDelegate::UpdateReferenceFrames(

media/gpu/vaapi/vp8_vaapi_video_encoder_delegate.h

@@ -32,13 +32,16 @@ class VP8VaapiVideoEncoderDelegate : public VaapiVideoEncoderDelegate {
     VideoBitrateAllocation bitrate_allocation;
 
     // Framerate in FPS.
-    uint32_t framerate;
+    uint32_t framerate = 0;
 
     // Quantization parameter. They are vp8 ac/dc indices and their ranges are
     // 0-127.
     uint8_t min_qp;
     uint8_t max_qp;
 
+    // The rate controller drop frame threshold. 0-100 as this is percentage.
+    uint8_t drop_frame_thresh = 0;
+
     // Error resilient mode.
     bool error_resilient_mode = false;
   };
@@ -64,7 +67,7 @@ class VP8VaapiVideoEncoderDelegate : public VaapiVideoEncoderDelegate {
  private:
   void InitializeFrameHeader();
 
-  void SetFrameHeader(
+  PrepareEncodeJobResult SetFrameHeader(
       size_t frame_num,
       VP8Picture& picture,
       std::array<bool, kNumVp8ReferenceBuffers>& ref_frames_used);