Record the Capture Mode (sensitivity level) in NetLog JSON

It is possible to infer the capture mode (which limits how much
sensitive information is stored in a NetLog file) by scanning emitted
events, but explicitly recording the capture mode in the file enables
automated tooling to more easily classify logs.

Bug: 1057773
Change-Id: Ia36fb2cbb091cf2b4a940fbe48fce87269438c33
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2087702
Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Eric Roman <eroman@chromium.org>
Commit-Queue: Eric Lawrence [MSFT] <ericlaw@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#815833}

components/cronet/cronet_url_request_context.cc

@@ -620,15 +620,15 @@ void CronetURLRequestContext::NetworkTasks::StartNetLog(
   // Do nothing if already logging to a file.
   if (net_log_file_observer_)
     return;
-  net_log_file_observer_ = net::FileNetLogObserver::CreateUnbounded(
-      file_path, /*constants=*/nullptr);
-  CreateNetLogEntriesForActiveObjects({context_.get()},
-                                      net_log_file_observer_.get());
+
   net::NetLogCaptureMode capture_mode =
       include_socket_bytes ? net::NetLogCaptureMode::kEverything
                            : net::NetLogCaptureMode::kDefault;
-  net_log_file_observer_->StartObserving(g_net_log.Get().net_log(),
-                                         capture_mode);
+  net_log_file_observer_ = net::FileNetLogObserver::CreateUnbounded(
+      file_path, capture_mode, /*constants=*/nullptr);
+  CreateNetLogEntriesForActiveObjects({context_.get()},
+                                      net_log_file_observer_.get());
+  net_log_file_observer_->StartObserving(g_net_log.Get().net_log());
 }
 
 void CronetURLRequestContext::NetworkTasks::StartNetLogToBoundedFile(
@@ -657,17 +657,16 @@ void CronetURLRequestContext::NetworkTasks::StartNetLogToBoundedFile(
     }
   }
 
+  net::NetLogCaptureMode capture_mode =
+      include_socket_bytes ? net::NetLogCaptureMode::kEverything
+                           : net::NetLogCaptureMode::kDefault;
   net_log_file_observer_ = net::FileNetLogObserver::CreateBounded(
-      file_path, size, /*constants=*/nullptr);
+      file_path, size, capture_mode, /*constants=*/nullptr);
 
   CreateNetLogEntriesForActiveObjects({context_.get()},
                                       net_log_file_observer_.get());
 
-  net::NetLogCaptureMode capture_mode =
-      include_socket_bytes ? net::NetLogCaptureMode::kEverything
-                           : net::NetLogCaptureMode::kDefault;
-  net_log_file_observer_->StartObserving(g_net_log.Get().net_log(),
-                                         capture_mode);
+  net_log_file_observer_->StartObserving(g_net_log.Get().net_log());
 }
 
 void CronetURLRequestContext::NetworkTasks::StopNetLog() {

components/cronet/ios/cronet_environment.mm

@@ -178,9 +178,8 @@ void CronetEnvironment::StartNetLogOnNetworkThread(const base::FilePath& path,
                 : net::NetLogCaptureMode::kDefault;
 
   file_net_log_observer_ =
-      net::FileNetLogObserver::CreateUnbounded(path, nullptr);
-  file_net_log_observer_->StartObserving(main_context_->net_log(),
-                                         capture_mode);
+      net::FileNetLogObserver::CreateUnbounded(path, capture_mode, nullptr);
+  file_net_log_observer_->StartObserving(main_context_->net_log());
   LOG(WARNING) << "Started NetLog";
 }
 

fuchsia/engine/browser/web_engine_net_log_observer.cc

@@ -42,8 +42,8 @@ WebEngineNetLogObserver::WebEngineNetLogObserver(
   if (!log_path.empty()) {
     net::NetLogCaptureMode capture_mode = net::NetLogCaptureMode::kDefault;
     file_net_log_observer_ = net::FileNetLogObserver::CreateUnbounded(
-        log_path, GetWebEngineConstants());
-    file_net_log_observer_->StartObserving(net::NetLog::Get(), capture_mode);
+        log_path, capture_mode, GetWebEngineConstants());
+    file_net_log_observer_->StartObserving(net::NetLog::Get());
   }
 }
 

google_apis/gcm/tools/mcs_probe.cc

@@ -338,10 +338,11 @@ void MCSProbe::UpdateCallback(bool success) {
 void MCSProbe::InitializeNetworkState() {
   if (command_line_.HasSwitch(kLogFileSwitch)) {
     base::FilePath log_path = command_line_.GetSwitchValuePath(kLogFileSwitch);
-    logger_ = net::FileNetLogObserver::CreateUnbounded(log_path, nullptr);
     net::NetLogCaptureMode capture_mode =
         net::NetLogCaptureMode::kIncludeSensitive;
-    logger_->StartObserving(net_log_, capture_mode);
+    logger_ = net::FileNetLogObserver::CreateUnbounded(log_path, capture_mode,
+                                                       nullptr);
+    logger_->StartObserving(net_log_);
   }
 
   net::URLRequestContextBuilder builder;

net/log/file_net_log_observer.cc

@@ -333,17 +333,19 @@ class FileNetLogObserver::FileWriter {
 std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateBounded(
     const base::FilePath& log_path,
     uint64_t max_total_size,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   return CreateInternal(log_path, SiblingInprogressDirectory(log_path),
                         base::nullopt, max_total_size, kDefaultNumFiles,
-                        std::move(constants));
+                        capture_mode, std::move(constants));
 }
 
 std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateUnbounded(
     const base::FilePath& log_path,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   return CreateInternal(log_path, base::FilePath(), base::nullopt, kNoLimit,
-                        kDefaultNumFiles, std::move(constants));
+                        kDefaultNumFiles, capture_mode, std::move(constants));
 }
 
 std::unique_ptr<FileNetLogObserver>
@@ -351,19 +353,23 @@ FileNetLogObserver::CreateBoundedPreExisting(
     const base::FilePath& inprogress_dir_path,
     base::File output_file,
     uint64_t max_total_size,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   return CreateInternal(base::FilePath(), inprogress_dir_path,
                         base::make_optional<base::File>(std::move(output_file)),
-                        max_total_size, kDefaultNumFiles, std::move(constants));
+                        max_total_size, kDefaultNumFiles, capture_mode,
+                        std::move(constants));
 }
 
 std::unique_ptr<FileNetLogObserver>
 FileNetLogObserver::CreateUnboundedPreExisting(
     base::File output_file,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   return CreateInternal(base::FilePath(), base::FilePath(),
                         base::make_optional<base::File>(std::move(output_file)),
-                        kNoLimit, kDefaultNumFiles, std::move(constants));
+                        kNoLimit, kDefaultNumFiles, capture_mode,
+                        std::move(constants));
 }
 
 FileNetLogObserver::~FileNetLogObserver() {
@@ -378,9 +384,8 @@ FileNetLogObserver::~FileNetLogObserver() {
   file_task_runner_->DeleteSoon(FROM_HERE, file_writer_.release());
 }
 
-void FileNetLogObserver::StartObserving(NetLog* net_log,
-                                        NetLogCaptureMode capture_mode) {
-  net_log->AddObserver(this, capture_mode);
+void FileNetLogObserver::StartObserving(NetLog* net_log) {
+  net_log->AddObserver(this, capture_mode_);
 }
 
 void FileNetLogObserver::StopObserving(std::unique_ptr<base::Value> polled_data,
@@ -425,10 +430,11 @@ std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateBoundedForTests(
     const base::FilePath& log_path,
     uint64_t max_total_size,
     size_t total_num_event_files,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   return CreateInternal(log_path, SiblingInprogressDirectory(log_path),
                         base::nullopt, max_total_size, total_num_event_files,
-                        std::move(constants));
+                        capture_mode, std::move(constants));
 }
 
 std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateInternal(
@@ -437,6 +443,7 @@ std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateInternal(
     base::Optional<base::File> pre_existing_log_file,
     uint64_t max_total_size,
     size_t total_num_event_files,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants) {
   DCHECK_GT(total_num_event_files, 0u);
 
@@ -469,25 +476,43 @@ std::unique_ptr<FileNetLogObserver> FileNetLogObserver::CreateInternal(
   return base::WrapUnique(new FileNetLogObserver(
       file_task_runner, std::move(file_writer),
       base::WrapRefCounted(new WriteQueue(write_queue_memory_max)),
-      std::move(constants)));
+      capture_mode, std::move(constants)));
 }
 
 FileNetLogObserver::FileNetLogObserver(
     scoped_refptr<base::SequencedTaskRunner> file_task_runner,
     std::unique_ptr<FileWriter> file_writer,
     scoped_refptr<WriteQueue> write_queue,
+    NetLogCaptureMode capture_mode,
     std::unique_ptr<base::Value> constants)
     : file_task_runner_(std::move(file_task_runner)),
       write_queue_(std::move(write_queue)),
-      file_writer_(std::move(file_writer)) {
+      file_writer_(std::move(file_writer)),
+      capture_mode_(capture_mode) {
   if (!constants)
     constants = base::Value::ToUniquePtrValue(GetNetConstants());
+
+  DCHECK(!constants->FindKey("logCaptureMode"));
+  constants->SetStringKey("logCaptureMode", CaptureModeToString(capture_mode));
   file_task_runner_->PostTask(
       FROM_HERE, base::BindOnce(&FileNetLogObserver::FileWriter::Initialize,
                                 base::Unretained(file_writer_.get()),
                                 std::move(constants)));
 }
 
+std::string FileNetLogObserver::CaptureModeToString(NetLogCaptureMode mode) {
+  switch (mode) {
+    case NetLogCaptureMode::kDefault:
+      return "Default";
+    case NetLogCaptureMode::kIncludeSensitive:
+      return "IncludeSensitive";
+    case NetLogCaptureMode::kEverything:
+      return "Everything";
+  }
+  NOTREACHED();
+  return "UNKNOWN";
+}
+
 FileNetLogObserver::WriteQueue::WriteQueue(uint64_t memory_max)
     : memory_(0), memory_max_(memory_max) {}
 

net/log/file_net_log_observer.h

@@ -58,11 +58,13 @@ class NET_EXPORT FileNetLogObserver : public NetLog::ThreadSafeObserver {
   static std::unique_ptr<FileNetLogObserver> CreateBounded(
       const base::FilePath& log_path,
       uint64_t max_total_size,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
   // Shortcut for calling CreateBounded() with kNoLimit.
   static std::unique_ptr<FileNetLogObserver> CreateUnbounded(
       const base::FilePath& log_path,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
   // Creates a bounded log that writes to a pre-existing file (truncating
@@ -73,18 +75,20 @@ class NET_EXPORT FileNetLogObserver : public NetLog::ThreadSafeObserver {
       const base::FilePath& inprogress_dir_path,
       base::File output_file,
       uint64_t max_total_size,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
   // Creates an unbounded log that writes to a pre-existing file (truncating
   // it to start with, and closing it upon completion).
   static std::unique_ptr<FileNetLogObserver> CreateUnboundedPreExisting(
       base::File output_file,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
   ~FileNetLogObserver() override;
 
   // Attaches this observer to |net_log| and begins observing events.
-  void StartObserving(NetLog* net_log, NetLogCaptureMode capture_mode);
+  void StartObserving(NetLog* net_log);
 
   // Stops observing net_log() and closes the output file(s). Must be called
   // after StartObserving. Should be called before destruction of the
@@ -112,6 +116,7 @@ class NET_EXPORT FileNetLogObserver : public NetLog::ThreadSafeObserver {
       const base::FilePath& log_path,
       uint64_t max_total_size,
       size_t total_num_event_files,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
  private:
@@ -124,13 +129,17 @@ class NET_EXPORT FileNetLogObserver : public NetLog::ThreadSafeObserver {
       base::Optional<base::File> pre_existing_out_file,
       uint64_t max_total_size,
       size_t total_num_event_files,
+      NetLogCaptureMode capture_mode,
       std::unique_ptr<base::Value> constants);
 
   FileNetLogObserver(scoped_refptr<base::SequencedTaskRunner> file_task_runner,
                      std::unique_ptr<FileWriter> file_writer,
                      scoped_refptr<WriteQueue> write_queue,
+                     NetLogCaptureMode capture_mode,
                      std::unique_ptr<base::Value> constants);
 
+  static std::string CaptureModeToString(NetLogCaptureMode mode);
+
   scoped_refptr<base::SequencedTaskRunner> file_task_runner_;
 
   // The |write_queue_| object is shared between the file task runner and the
@@ -147,6 +156,8 @@ class NET_EXPORT FileNetLogObserver : public NetLog::ThreadSafeObserver {
   // finished (since it is posted using base::Unretained()).
   std::unique_ptr<FileWriter> file_writer_;
 
+  const NetLogCaptureMode capture_mode_;
+
   DISALLOW_COPY_AND_ASSIGN(FileNetLogObserver);
 };
 

net/log/file_net_log_observer_unittest.cc

@@ -234,16 +234,19 @@ class FileNetLogObserverTest : public ::testing::TestWithParam<bool>,
 
   bool IsBounded() const { return GetParam(); }
 
-  void CreateAndStartObserving(std::unique_ptr<base::Value> constants) {
+  void CreateAndStartObserving(
+      std::unique_ptr<base::Value> constants,
+      NetLogCaptureMode capture_mode = NetLogCaptureMode::kDefault) {
     if (IsBounded()) {
       logger_ = FileNetLogObserver::CreateBoundedForTests(
-          log_path_, kLargeFileSize, kTotalNumFiles, std::move(constants));
+          log_path_, kLargeFileSize, kTotalNumFiles, capture_mode,
+          std::move(constants));
     } else {
-      logger_ =
-          FileNetLogObserver::CreateUnbounded(log_path_, std::move(constants));
+      logger_ = FileNetLogObserver::CreateUnbounded(log_path_, capture_mode,
+                                                    std::move(constants));
     }
 
-    logger_->StartObserving(&net_log_, NetLogCaptureMode::kDefault);
+    logger_->StartObserving(&net_log_);
   }
 
   void CreateAndStartObservingPreExisting(
@@ -259,13 +262,13 @@ class FileNetLogObserverTest : public ::testing::TestWithParam<bool>,
     if (IsBounded()) {
       logger_ = FileNetLogObserver::CreateBoundedPreExisting(
           scratch_dir_.GetPath(), std::move(file), kLargeFileSize,
-          std::move(constants));
+          NetLogCaptureMode::kDefault, std::move(constants));
     } else {
       logger_ = FileNetLogObserver::CreateUnboundedPreExisting(
-          std::move(file), std::move(constants));
+          std::move(file), NetLogCaptureMode::kDefault, std::move(constants));
     }
 
-    logger_->StartObserving(&net_log_, NetLogCaptureMode::kDefault);
+    logger_->StartObserving(&net_log_);
   }
 
   bool LogFileExists() {
@@ -303,8 +306,9 @@ class FileNetLogObserverBoundedTest : public ::testing::Test,
                                uint64_t total_file_size,
                                int num_files) {
     logger_ = FileNetLogObserver::CreateBoundedForTests(
-        log_path_, total_file_size, num_files, std::move(constants));
-    logger_->StartObserving(&net_log_, NetLogCaptureMode::kDefault);
+        log_path_, total_file_size, num_files, NetLogCaptureMode::kDefault,
+        std::move(constants));
+    logger_->StartObserving(&net_log_);
   }
 
   // Returns the path for an internally directory created for bounded logs (this
@@ -486,11 +490,12 @@ TEST_P(FileNetLogObserverTest, PreExistingFileBroken) {
   EXPECT_FALSE(file.IsValid());
   if (IsBounded())
     logger_ = FileNetLogObserver::CreateBoundedPreExisting(
-        scratch_dir_.GetPath(), std::move(file), kLargeFileSize, nullptr);
+        scratch_dir_.GetPath(), std::move(file), kLargeFileSize,
+        NetLogCaptureMode::kDefault, nullptr);
   else
-    logger_ = FileNetLogObserver::CreateUnboundedPreExisting(std::move(file),
-                                                             nullptr);
-  logger_->StartObserving(&net_log_, NetLogCaptureMode::kDefault);
+    logger_ = FileNetLogObserver::CreateUnboundedPreExisting(
+        std::move(file), NetLogCaptureMode::kDefault, nullptr);
+  logger_->StartObserving(&net_log_);
 
   // Send dummy event.
   AddEntries(logger_.get(), 1, kDummyEventSize);
@@ -551,6 +556,28 @@ TEST_P(FileNetLogObserverTest, GeneratesValidJSONWithPolledData) {
                                    kDummyPolledDataString);
 }
 
+// Ensure that the Capture Mode is recorded as a constant in the NetLog.
+TEST_P(FileNetLogObserverTest, LogModeRecorded) {
+  struct TestCase {
+    NetLogCaptureMode capture_mode;
+    const char* expected_value;
+  } test_cases[] = {// Challenges that result in success results.
+                    {NetLogCaptureMode::kEverything, "Everything"},
+                    {NetLogCaptureMode::kIncludeSensitive, "IncludeSensitive"},
+                    {NetLogCaptureMode::kDefault, "Default"}};
+
+  TestClosure closure;
+  for (const auto& test_case : test_cases) {
+    CreateAndStartObserving(nullptr, test_case.capture_mode);
+    logger_->StopObserving(nullptr, closure.closure());
+    closure.WaitForResult();
+    std::unique_ptr<ParsedNetLog> log = ReadNetLogFromDisk(log_path_);
+    ASSERT_TRUE(log);
+    ExpectDictionaryContainsProperty(log->constants, "logCaptureMode",
+                                     test_case.expected_value);
+  }
+}
+
 // Adds events concurrently from several different threads. The exact order of
 // events seen by this test is non-deterministic.
 TEST_P(FileNetLogObserverTest, AddEventsFromMultipleThreads) {
@@ -946,8 +973,9 @@ TEST_F(FileNetLogObserverBoundedTest, PreExistingUsesSpecifiedDir) {
   file.Write(0, "not json", 8);
 
   logger_ = FileNetLogObserver::CreateBoundedPreExisting(
-      scratch_dir.GetPath(), std::move(file), kLargeFileSize, nullptr);
-  logger_->StartObserving(&net_log_, NetLogCaptureMode::kDefault);
+      scratch_dir.GetPath(), std::move(file), kLargeFileSize,
+      NetLogCaptureMode::kDefault, nullptr);
+  logger_->StartObserving(&net_log_);
 
   base::ThreadPoolInstance::Get()->FlushForTesting();
   EXPECT_TRUE(base::PathExists(log_path_));

services/network/net_log_exporter.cc

@@ -185,12 +185,12 @@ void NetLogExporter::StartWithScratchDir(
 
   if (max_file_size != kUnlimitedFileSize) {
     file_net_observer_ = net::FileNetLogObserver::CreateBoundedPreExisting(
-        scratch_dir_path, std::move(destination_), max_file_size,
+        scratch_dir_path, std::move(destination_), max_file_size, capture_mode,
         std::move(constants));
   } else {
     DCHECK(scratch_dir_path.empty());
     file_net_observer_ = net::FileNetLogObserver::CreateUnboundedPreExisting(
-        std::move(destination_), std::move(constants));
+        std::move(destination_), capture_mode, std::move(constants));
   }
 
   // There might not be a NetworkService object e.g. on iOS; in that case
@@ -206,7 +206,7 @@ void NetLogExporter::StartWithScratchDir(
   }
 
   file_net_observer_->StartObserving(
-      network_context_->url_request_context()->net_log(), capture_mode);
+      network_context_->url_request_context()->net_log());
   std::move(callback).Run(net::OK);
 }
 

services/network/network_service.cc

@@ -482,8 +482,8 @@ void NetworkService::StartNetLog(base::File file,
   constants->MergeDictionary(&client_constants);
 
   file_net_log_observer_ = net::FileNetLogObserver::CreateUnboundedPreExisting(
-      std::move(file), std::move(constants));
-  file_net_log_observer_->StartObserving(net_log_, capture_mode);
+      std::move(file), capture_mode, std::move(constants));
+  file_net_log_observer_->StartObserving(net_log_);
 }
 
 void NetworkService::AttachNetLogProxy(