Allow running inspector protocol tests using protocol logs

This CL implements a flag called `--inspector-protocol-log` that accepts
a path to a Chrome DevTools Protocol message log. If specified, the test
runner would replay the log mocking the actual browser. The purpose of
this flag is to allow reproducing test flakiness locally and it is not
meant to be used on the bots for now.

Bug: 327140253
Change-Id: I871442d568878b3a1a0d71a18eb2eb721b457e76
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5331573
Reviewed-by: danakj <danakj@chromium.org>
Commit-Queue: Alex Rudenko <alexrudenko@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1274969}

content/web_test/browser/devtools_protocol_test_bindings.cc

@@ -8,8 +8,10 @@
 #include "base/functional/bind.h"
 #include "base/functional/callback_helpers.h"
 #include "base/json/json_reader.h"
+#include "base/json/json_writer.h"
 #include "base/json/string_escape.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/values.h"
 #include "build/build_config.h"
@@ -36,12 +38,14 @@ constexpr size_t kWebTestMaxMessageChunkSize =
 }  // namespace
 
 DevToolsProtocolTestBindings::DevToolsProtocolTestBindings(
-    WebContents* devtools)
+    WebContents* devtools,
+    std::string log)
     : WebContentsObserver(devtools),
       agent_host_(DevToolsAgentHost::CreateForBrowser(
           nullptr,
           DevToolsAgentHost::CreateServerSocketCallback())) {
   agent_host_->AttachClient(this);
+  ParseLog(log);
 }
 
 DevToolsProtocolTestBindings::~DevToolsProtocolTestBindings() {
@@ -71,6 +75,20 @@ GURL DevToolsProtocolTestBindings::MapTestURLIfNeeded(const GURL& test_url,
   return GURL(spec);
 }
 
+void DevToolsProtocolTestBindings::ParseLog(const std::string_view log) {
+  if (log.empty()) {
+    return;
+  }
+  std::vector<std::string> lines = base::SplitStringUsingSubstr(
+      log, "\n", base::TRIM_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
+  for (const std::string& line : lines) {
+    std::optional<base::Value::Dict> item = base::JSONReader::ReadDict(line);
+    CHECK(!item->empty());
+    log_.push_back(std::move(item.value()));
+  }
+  log_enabled_ = true;
+}
+
 void DevToolsProtocolTestBindings::ReadyToCommitNavigation(
     NavigationHandle* navigation_handle) {
 #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
@@ -91,6 +109,39 @@ void DevToolsProtocolTestBindings::WebContentsDestroyed() {
   }
 }
 
+void DevToolsProtocolTestBindings::HandleMessagesFromLog(
+    const std::string_view protocol_message_string) {
+  std::optional<base::Value::Dict> parsed =
+      base::JSONReader::ReadDict(protocol_message_string);
+  if (!parsed) {
+    return;
+  }
+  base::Value::Dict protocol_message = std::move(parsed.value());
+
+  CHECK(log_pos_ < log_.size()) << "Test sent commands but the log is empty";
+  const base::Value::Dict& top = log_[log_pos_];
+  CHECK(protocol_message == top)
+      << "Test sent a command that is not the next in the log \n"
+      << protocol_message << "\n"
+      << top;
+  log_pos_++;
+  while (log_pos_ < log_.size()) {
+    const base::Value::Dict& item = log_[log_pos_];
+    // Stop when the next command is encountered in the log.
+    if (item.FindString("method") && item.FindInt("id")) {
+      break;
+    }
+    log_pos_++;
+    std::optional<std::string> str_message = base::WriteJson(item);
+    CHECK(str_message) << "Could not convert log message to JSON";
+    std::string param;
+    base::EscapeJSONString(str_message.value(), true, &param);
+    std::string javascript = "DevToolsAPI.dispatchMessage(" + param + ");";
+    web_contents()->GetPrimaryMainFrame()->ExecuteJavaScriptForTests(
+        base::UTF8ToUTF16(javascript), base::NullCallback());
+  }
+}
+
 void DevToolsProtocolTestBindings::HandleMessageFromTest(
     base::Value::Dict message) {
   const std::string* method = message.FindString("method");
@@ -103,6 +154,11 @@ void DevToolsProtocolTestBindings::HandleMessageFromTest(
     if (!protocol_message)
       return;
 
+    if (log_enabled_) {
+      HandleMessagesFromLog(*protocol_message);
+      return;
+    }
+
     if (agent_host_) {
       WebTestControlHost::Get()->PrintMessageToStderr(
           "Protocol message: " + *protocol_message + "\n");
@@ -116,6 +172,9 @@ void DevToolsProtocolTestBindings::HandleMessageFromTest(
 void DevToolsProtocolTestBindings::DispatchProtocolMessage(
     DevToolsAgentHost* agent_host,
     base::span<const uint8_t> message) {
+  if (log_enabled_) {
+    NOTREACHED_NORETURN() << "Unexpected messages dispatched by the browser";
+  }
   base::StringPiece str_message(reinterpret_cast<const char*>(message.data()),
                                 message.size());
   WebTestControlHost::Get()->PrintMessageToStderr(

content/web_test/browser/devtools_protocol_test_bindings.h

@@ -20,7 +20,7 @@ class DevToolsFrontendHost;
 class DevToolsProtocolTestBindings : public WebContentsObserver,
                                      public DevToolsAgentHostClient {
  public:
-  explicit DevToolsProtocolTestBindings(WebContents* devtools);
+  explicit DevToolsProtocolTestBindings(WebContents* devtools, std::string log);
 
   DevToolsProtocolTestBindings(const DevToolsProtocolTestBindings&) = delete;
   DevToolsProtocolTestBindings& operator=(const DevToolsProtocolTestBindings&) =
@@ -40,6 +40,8 @@ class DevToolsProtocolTestBindings : public WebContentsObserver,
   void ReadyToCommitNavigation(NavigationHandle* navigation_handle) override;
   void WebContentsDestroyed() override;
 
+  void ParseLog(const std::string_view log);
+  void HandleMessagesFromLog(const std::string_view protocol_message_string);
   void HandleMessageFromTest(base::Value::Dict message);
 
   scoped_refptr<DevToolsAgentHost> agent_host_;
@@ -48,6 +50,13 @@ class DevToolsProtocolTestBindings : public WebContentsObserver,
   // run web tests natively on Android.
   std::unique_ptr<DevToolsFrontendHost> frontend_host_;
 #endif
+  // Log of protocol messages, used to script the bindings behavior.
+  std::vector<base::Value::Dict> log_;
+  // The index of the next message in the log.
+  size_t log_pos_ = 0;
+  // If true, the binding is using the log instead of sending real messages.
+  // The log is enabled if a non-empty log is provided via the constructor.
+  bool log_enabled_ = false;
 };
 
 }  // namespace content

content/web_test/browser/web_test_control_host.cc

@@ -3,6 +3,8 @@
 // found in the LICENSE file.
 
 #include "content/web_test/browser/web_test_control_host.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
 #include "base/memory/raw_ptr.h"
 
 #include <stddef.h>
@@ -624,9 +626,23 @@ void WebTestControlHost::PrepareForWebTest(const TestInfo& test_info) {
   HandleNewRenderFrameHost(main_window_->web_contents()->GetPrimaryMainFrame());
 
   if (is_devtools_protocol_test) {
+    std::string log;
+    if (base::CommandLine::ForCurrentProcess()->HasSwitch(
+            switches::kInspectorProtocolLog)) {
+      base::FilePath log_path =
+          base::CommandLine::ForCurrentProcess()->GetSwitchValuePath(
+              switches::kInspectorProtocolLog);
+      base::ScopedAllowBlockingForTesting allow_blocking;
+      if (!base::ReadFileToString(log_path, &log)) {
+        printer_->AddErrorMessage(base::StringPrintf(
+            "FAIL: Failed to read the inspector-protocol-log file %s",
+            log_path.AsUTF8Unsafe().c_str()));
+      }
+    }
+
     devtools_protocol_test_bindings_ =
         std::make_unique<DevToolsProtocolTestBindings>(
-            main_window_->web_contents());
+            main_window_->web_contents(), log);
   }
 
   // We don't go down the normal system path of focusing RenderWidgetHostView

content/web_test/common/web_test_switches.cc

@@ -29,6 +29,13 @@ const char kAlwaysUseComplexText[] = "always-use-complex-text";
 // whether or not reloading a webpage releases web-related objects correctly.
 const char kEnableLeakDetection[] = "enable-leak-detection";
 
+// Specifies the path to a file containing a Chrome DevTools protocol log.
+// Each line in the log file is expected to be a protocol message in the JSON
+// format. The test runner will use this log file to script the backend for any
+// inspector-protocol tests that run. Usually you would want to run a single
+// test using the log to reproduce timeouts or crashes.
+const char kInspectorProtocolLog[] = "inspector-protocol-log";
+
 // Encode binary web test results (images, audio) using base64.
 const char kEncodeBinary[] = "encode-binary";
 

content/web_test/common/web_test_switches.h

@@ -14,6 +14,7 @@ extern const char kEnableAccelerated2DCanvas[];
 extern const char kEnableFontAntialiasing[];
 extern const char kAlwaysUseComplexText[];
 extern const char kEnableLeakDetection[];
+extern const char kInspectorProtocolLog[];
 extern const char kEncodeBinary[];
 extern const char kStableReleaseMode[];
 extern const char kDisableHeadlessMode[];

docs/testing/web_tests.md

@@ -614,6 +614,27 @@ NOTE: If the test is an html file, this means it's a legacy test so you need to
   }
   ```
 
+### Reproducing flaky inspector protocol tests
+
+https://crrev.com/c/5318502 implemented logging for inspector-protocol tests.
+With this CL for each test in stderr you should see Chrome DevTools Protocol
+messages that the test and the browser exchanged.
+
+You can use this log to reproduce the failure or timeout locally.
+
+* Prepare a log file and ensure each line contains one protocol message
+in the JSON format. Strip any prefixes or non-protocol messages from the
+original log.
+* Make sure your local test file version matches the version that produced
+the log file.
+* Run the test using the log file:
+
+  ```sh
+  third_party/blink/tools/run_web_tests.py -t Release \
+   --additional-driver-flag="--inspector-protocol-log=/path/to/log.txt" \
+   http/tests/inspector-protocol/network/url-fragment.js
+  ```
+
 ## Bisecting Regressions
 
 You can use [`git bisect`](https://git-scm.com/docs/git-bisect) to find which