media/gpu/video_encode_accelerator_perf_tests: Add MeasureProducedBitstreamQuality test case

This adds the test case to video_encode_accelerator_perf_tests
that compute SSIM and PSNR values of the bitstream produced by
VideoEncodeAccelerator.

Bug: b:174322759
Test: video_encode_accelerator_tests on atlas
Change-Id: I78817f1a91d926b3041a1c8e02a2d70c4bbf828e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2563012
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: David Staessens <dstaessens@chromium.org>
Cr-Commit-Position: refs/heads/master@{#835986}

media/gpu/BUILD.gn

@@ -619,9 +619,12 @@ if (use_v4l2_codec || use_vaapi) {
     data = [ "//media/test/data/" ]
     deps = [
       ":buildflags",
+      "test:frame_validator",
       "test:helpers",
+      "test:test_helpers",
       "test:video_encoder",
       "test:video_encoder_test_environment",
+      "test:video_encoder_test_environment",
       "//media:test_support",
       "//testing/gtest",
     ]

media/gpu/test/video_frame_validator.h

@@ -184,7 +184,7 @@ class RawVideoFrameValidator : public VideoFrameValidator {
 };
 
 // Validate by computing PSNR from the frame to be validated and the model frame
-// acquired by |get_model_frame_cb_|. If the PSNR value is equal to or less than
+// acquired by |get_model_frame_cb_|. If the PSNR value is equal to or more than
 // |tolerance_|, the validation on the frame passes.
 class PSNRVideoFrameValidator : public VideoFrameValidator {
  public:
@@ -195,7 +195,7 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr,
       ValidationMode validation_mode = ValidationMode::kThreshold,
       double tolerance = kDefaultTolerance);
-  const std::map<size_t, double>& GetPSNRValues() { return psnr_; }
+  const std::map<size_t, double>& GetPSNRValues() const { return psnr_; }
   ~PSNRVideoFrameValidator() override;
 
  private:
@@ -220,7 +220,7 @@ class PSNRVideoFrameValidator : public VideoFrameValidator {
 };
 
 // Validate by computing SSIM from the frame to be validated and the model frame
-// acquired by |get_model_frame_cb_|. If the SSIM value is equal to or less than
+// acquired by |get_model_frame_cb_|. If the SSIM value is equal to or more than
 // |tolerance_|, the validation on the frame passes.
 class SSIMVideoFrameValidator : public VideoFrameValidator {
  public:
@@ -231,7 +231,7 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
       std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr,
       ValidationMode validation_mode = ValidationMode::kThreshold,
       double tolerance = kDefaultTolerance);
-  const std::map<size_t, double>& GetSSIMValues() { return ssim_; }
+  const std::map<size_t, double>& GetSSIMValues() const { return ssim_; }
   ~SSIMVideoFrameValidator() override;
 
  private:

media/gpu/video_encode_accelerator_perf_tests.cc

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include <algorithm>
+#include <map>
 #include <numeric>
 #include <vector>
 
@@ -11,11 +12,16 @@
 #include "base/json/json_writer.h"
 #include "base/strings/stringprintf.h"
 #include "media/base/bitstream_buffer.h"
+#include "media/base/media_util.h"
 #include "media/base/test_data_util.h"
+#include "media/base/video_decoder_config.h"
 #include "media/gpu/test/video.h"
+#include "media/gpu/test/video_encoder/bitstream_validator.h"
 #include "media/gpu/test/video_encoder/video_encoder.h"
 #include "media/gpu/test/video_encoder/video_encoder_client.h"
 #include "media/gpu/test/video_encoder/video_encoder_test_environment.h"
+#include "media/gpu/test/video_frame_validator.h"
+#include "media/gpu/test/video_test_helpers.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace media {
@@ -268,21 +274,194 @@ void PerformanceMetrics::WriteToFile() const {
   VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
 }
 
+struct BitstreamQualityMetrics {
+  BitstreamQualityMetrics(const PSNRVideoFrameValidator& psnr_validator,
+                          const SSIMVideoFrameValidator& ssim_validator);
+  void WriteToConsole() const;
+  void WriteToFile() const;
+
+ private:
+  struct QualityStats {
+    double avg = 0;
+    double percentile_25 = 0;
+    double percentile_50 = 0;
+    double percentile_75 = 0;
+    std::vector<double> values_in_order;
+  };
+
+  static QualityStats ComputeQualityStats(
+      const std::map<size_t, double>& values);
+
+  const QualityStats psnr_stats;
+  const QualityStats ssim_stats;
+};
+
+BitstreamQualityMetrics::BitstreamQualityMetrics(
+    const PSNRVideoFrameValidator& psnr_validator,
+    const SSIMVideoFrameValidator& ssim_validator)
+    : psnr_stats(ComputeQualityStats(psnr_validator.GetPSNRValues())),
+      ssim_stats(ComputeQualityStats(ssim_validator.GetSSIMValues())) {}
+
+// static
+BitstreamQualityMetrics::QualityStats
+BitstreamQualityMetrics::ComputeQualityStats(
+    const std::map<size_t, double>& values) {
+  if (values.empty())
+    return QualityStats();
+  std::vector<double> sorted_values;
+  std::vector<std::pair<size_t, double>> index_and_value;
+  sorted_values.reserve(values.size());
+  index_and_value.reserve(values.size());
+  for (const auto& v : values) {
+    sorted_values.push_back(v.second);
+    index_and_value.emplace_back(v.first, v.second);
+  }
+  std::sort(sorted_values.begin(), sorted_values.end());
+  std::sort(index_and_value.begin(), index_and_value.end());
+  QualityStats stats;
+  stats.avg = std::accumulate(sorted_values.begin(), sorted_values.end(), 0.0) /
+              sorted_values.size();
+  stats.percentile_25 = sorted_values[sorted_values.size() / 4];
+  stats.percentile_50 = sorted_values[sorted_values.size() / 2];
+  stats.percentile_75 = sorted_values[(sorted_values.size() * 3) / 4];
+  stats.values_in_order.resize(index_and_value.size());
+  for (size_t i = 0; i < index_and_value.size(); ++i)
+    stats.values_in_order[i] = index_and_value[i].second;
+  return stats;
+}
+
+void BitstreamQualityMetrics::WriteToConsole() const {
+  std::cout << "SSIM - average:       " << ssim_stats.avg << std::endl;
+  std::cout << "SSIM - percentile 25: " << ssim_stats.percentile_25
+            << std::endl;
+  std::cout << "SSIM - percentile 50: " << ssim_stats.percentile_50
+            << std::endl;
+  std::cout << "SSIM - percentile 75: " << ssim_stats.percentile_75
+            << std::endl;
+  std::cout << "PSNR - average:       " << psnr_stats.avg << std::endl;
+  std::cout << "PSNR - percentile 25: " << psnr_stats.percentile_25
+            << std::endl;
+  std::cout << "PSNR - percentile 50: " << psnr_stats.percentile_50
+            << std::endl;
+  std::cout << "PSNR - percentile 75: " << psnr_stats.percentile_75
+            << std::endl;
+}
+
+void BitstreamQualityMetrics::WriteToFile() const {
+  base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
+  if (!DirectoryExists(output_folder_path))
+    base::CreateDirectory(output_folder_path);
+  output_folder_path = base::MakeAbsoluteFilePath(output_folder_path);
+  // Write quality metrics to json.
+  base::Value metrics(base::Value::Type::DICTIONARY);
+  metrics.SetKey("SSIMAverage", base::Value(ssim_stats.avg));
+  metrics.SetKey("SSIMPercentile25", base::Value(ssim_stats.percentile_25));
+  metrics.SetKey("SSIMPercentile50", base::Value(ssim_stats.percentile_50));
+  metrics.SetKey("SSIMPercentile75", base::Value(psnr_stats.percentile_75));
+  metrics.SetKey("PSNRAverage", base::Value(psnr_stats.avg));
+  metrics.SetKey("PSNRPercentile25", base::Value(psnr_stats.percentile_25));
+  metrics.SetKey("PSNRPercentile50", base::Value(psnr_stats.percentile_50));
+  metrics.SetKey("PSNRPercentile75", base::Value(psnr_stats.percentile_75));
+  // Write ssim values bitstream delivery times to json.
+  base::Value ssim_values(base::Value::Type::LIST);
+  for (double value : ssim_stats.values_in_order)
+    ssim_values.Append(value);
+  metrics.SetKey("SSIMValues", std::move(ssim_values));
+
+  // Write psnr values to json.
+  base::Value psnr_values(base::Value::Type::LIST);
+  for (double value : psnr_stats.values_in_order)
+    psnr_values.Append(value);
+  metrics.SetKey("PSNRValues", std::move(psnr_values));
+
+  // Write json to file.
+  std::string metrics_str;
+  ASSERT_TRUE(base::JSONWriter::WriteWithOptions(
+      metrics, base::JSONWriter::OPTIONS_PRETTY_PRINT, &metrics_str));
+  base::FilePath metrics_file_path = output_folder_path.Append(
+      g_env->GetTestOutputFilePath().AddExtension(FILE_PATH_LITERAL(".json")));
+  // Make sure that the directory into which json is saved is created.
+  LOG_ASSERT(base::CreateDirectory(metrics_file_path.DirName()));
+  base::File metrics_output_file(
+      base::FilePath(metrics_file_path),
+      base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE);
+  int bytes_written = metrics_output_file.WriteAtCurrentPos(
+      metrics_str.data(), metrics_str.length());
+  ASSERT_EQ(bytes_written, static_cast<int>(metrics_str.length()));
+  VLOG(0) << "Wrote performance metrics to: " << metrics_file_path;
+}
+
 // Video encode test class. Performs setup and teardown for each single test.
 class VideoEncoderTest : public ::testing::Test {
  public:
   // Create a new video encoder instance.
-  std::unique_ptr<VideoEncoder> CreateVideoEncoder(const Video* video,
+  std::unique_ptr<VideoEncoder> CreateVideoEncoder(Video* video,
                                                    VideoCodecProfile profile,
                                                    uint32_t bitrate,
                                                    uint32_t encoder_rate = 0) {
-    LOG_ASSERT(video);
-
-    std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
     auto performance_evaluator = std::make_unique<PerformanceEvaluator>();
     performance_evaluator_ = performance_evaluator.get();
+    std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
     bitstream_processors.push_back(std::move(performance_evaluator));
+    return CreateVideoEncoderWithProcessors(
+        video, profile, bitrate, encoder_rate, std::move(bitstream_processors));
+  }
 
+  // Create a new video encoder instance for quality performance tests.
+  std::unique_ptr<VideoEncoder> CreateVideoEncoderForQualityPerformance(
+      Video* video,
+      VideoCodecProfile profile,
+      uint32_t bitrate) {
+    raw_data_helper_ = RawDataHelper::Create(video);
+    if (!raw_data_helper_) {
+      LOG(ERROR) << "Failed to create raw data helper";
+      return nullptr;
+    }
+
+    std::vector<std::unique_ptr<VideoFrameProcessor>> video_frame_processors;
+    VideoFrameValidator::GetModelFrameCB get_model_frame_cb =
+        base::BindRepeating(&VideoEncoderTest::GetModelFrame,
+                            base::Unretained(this));
+    auto ssim_validator = SSIMVideoFrameValidator::Create(
+        get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
+        VideoFrameValidator::ValidationMode::kAverage,
+        /*tolerance=*/0.0);
+    LOG_ASSERT(ssim_validator);
+    ssim_validator_ = ssim_validator.get();
+    video_frame_processors.push_back(std::move(ssim_validator));
+    auto psnr_validator = PSNRVideoFrameValidator::Create(
+        get_model_frame_cb, /*corrupt_frame_processor=*/nullptr,
+        VideoFrameValidator::ValidationMode::kAverage,
+        /*tolerance=*/0.0);
+    LOG_ASSERT(psnr_validator);
+    psnr_validator_ = psnr_validator.get();
+    video_frame_processors.push_back(std::move(psnr_validator));
+
+    const gfx::Rect visible_rect(video->Resolution());
+    VideoDecoderConfig decoder_config(
+        VideoCodecProfileToVideoCodec(profile), profile,
+        VideoDecoderConfig::AlphaMode::kIsOpaque, VideoColorSpace(),
+        kNoTransformation, visible_rect.size(), visible_rect,
+        visible_rect.size(), EmptyExtraData(), EncryptionScheme::kUnencrypted);
+
+    auto bitstream_validator = BitstreamValidator::Create(
+        decoder_config, kNumFramesToEncodeForPerformance - 1,
+        std::move(video_frame_processors));
+    LOG_ASSERT(bitstream_validator);
+    std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors;
+    bitstream_processors.push_back(std::move(bitstream_validator));
+    return CreateVideoEncoderWithProcessors(video, profile, bitrate,
+                                            /*encoder_rate=*/0,
+                                            std::move(bitstream_processors));
+  }
+
+  std::unique_ptr<VideoEncoder> CreateVideoEncoderWithProcessors(
+      Video* video,
+      VideoCodecProfile profile,
+      uint32_t bitrate,
+      uint32_t encoder_rate,
+      std::vector<std::unique_ptr<BitstreamProcessor>> bitstream_processors) {
+    LOG_ASSERT(video);
     constexpr size_t kNumTemporalLayers = 1u;
     VideoEncoderClientConfig config(video, profile, kNumTemporalLayers,
                                     bitrate);
@@ -290,6 +469,7 @@ class VideoEncoderTest : public ::testing::Test {
     if (encoder_rate != 0)
       config.encode_interval =
           base::TimeDelta::FromSeconds(/*secs=*/1u) / encoder_rate;
+
     auto video_encoder =
         VideoEncoder::Create(config, g_env->GetGpuMemoryBufferFactory(),
                              std::move(bitstream_processors));
@@ -299,7 +479,17 @@ class VideoEncoderTest : public ::testing::Test {
     return video_encoder;
   }
 
+  scoped_refptr<const VideoFrame> GetModelFrame(size_t frame_index) {
+    LOG_ASSERT(raw_data_helper_);
+    return raw_data_helper_->GetFrame(frame_index %
+                                      g_env->Video()->NumFrames());
+  }
+
+  std::unique_ptr<RawDataHelper> raw_data_helper_;
+
   PerformanceEvaluator* performance_evaluator_;
+  SSIMVideoFrameValidator* ssim_validator_;
+  PSNRVideoFrameValidator* psnr_validator_;
 };
 
 }  // namespace
@@ -343,6 +533,20 @@ TEST_F(VideoEncoderTest, MeasureCappedPerformance) {
   EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
   EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
 }
+
+TEST_F(VideoEncoderTest, MeasureProducedBitstreamQuality) {
+  auto encoder = CreateVideoEncoderForQualityPerformance(
+      g_env->Video(), g_env->Profile(), g_env->Bitrate());
+  encoder->Encode();
+  EXPECT_TRUE(encoder->WaitForFlushDone());
+  EXPECT_EQ(encoder->GetFlushDoneCount(), 1u);
+  EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
+  EXPECT_TRUE(encoder->WaitForBitstreamProcessors());
+
+  BitstreamQualityMetrics metrics(*psnr_validator_, *ssim_validator_);
+  metrics.WriteToConsole();
+  metrics.WriteToFile();
+}
 }  // namespace test
 }  // namespace media