Dump stats of memory pages which are hooked, but absent from /proc/<pid>/maps.

BUG=189114


Review URL: https://chromiumcodereview.appspot.com/12575008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@188272 0039d316-1c4b-4281-b951-d872f2087c98

third_party/tcmalloc/chromium/src/deep-heap-profile.cc

@@ -704,6 +704,10 @@ void DeepHeapProfile::GlobalStats::SnapshotMaps(
         }
 
         if (last_address_of_unhooked + 1 > cursor) {
+          RAW_CHECK(cursor >= first_address,
+                    "Wrong calculation for unhooked");
+          RAW_CHECK(last_address_of_unhooked <= last_address,
+                    "Wrong calculation for unhooked");
           uint64 committed_size = unhooked_[type].Record(
               memory_residence_info_getter,
               cursor,
@@ -748,6 +752,37 @@ void DeepHeapProfile::GlobalStats::SnapshotMaps(
                mmap_iter->end_addr - 1 <= last_address);
     }
   }
+
+  // TODO(dmikurube): Investigate and fix http://crbug.com/189114.
+  //
+  // The total committed memory usage in all_ (from /proc/<pid>/maps) is
+  // sometimes smaller than the sum of the committed mmap'ed addresses and
+  // unhooked regions.  Within our observation, the difference was only 4KB
+  // in committed usage, zero in reserved virtual addresses
+  //
+  // A guess is that an uncommitted (but reserved) page may become committed
+  // during counting memory usage in the loop above.
+  //
+  // The difference is accounted as "ABSENT" to investigate such cases.
+
+  RegionStats all_total;
+  RegionStats unhooked_total;
+  for (int i = 0; i < NUMBER_OF_MAPS_REGION_TYPES; ++i) {
+    all_total.AddAnotherRegionStat(all_[i]);
+    unhooked_total.AddAnotherRegionStat(unhooked_[i]);
+  }
+
+  size_t absent_virtual = profiled_mmap_.virtual_bytes() +
+                          unhooked_total.virtual_bytes() -
+                          all_total.virtual_bytes();
+  if (absent_virtual > 0)
+    all_[ABSENT].AddToVirtualBytes(absent_virtual);
+
+  size_t absent_committed = profiled_mmap_.committed_bytes() +
+                            unhooked_total.committed_bytes() -
+                            all_total.committed_bytes();
+  if (absent_committed > 0)
+    all_[ABSENT].AddToCommittedBytes(absent_committed);
 }
 
 void DeepHeapProfile::GlobalStats::SnapshotAllocations(
@@ -786,6 +821,7 @@ void DeepHeapProfile::GlobalStats::Unparse(TextBuffer* buffer) {
   buffer->AppendString("\n", 0);
 
   all_total.Unparse("total", buffer);
+  all_[ABSENT].Unparse("absent", buffer);
   all_[FILE_EXEC].Unparse("file-exec", buffer);
   all_[FILE_NONEXEC].Unparse("file-nonexec", buffer);
   all_[ANONYMOUS].Unparse("anonymous", buffer);

third_party/tcmalloc/chromium/src/deep-heap-profile.h

@@ -230,6 +230,7 @@ class DeepHeapProfile {
     // Writes stats of the region into |buffer| with |name|.
     void Unparse(const char* name, TextBuffer* buffer);
 
+    size_t virtual_bytes() const { return virtual_bytes_; }
     size_t committed_bytes() const { return committed_bytes_; }
     void AddToVirtualBytes(size_t additional_virtual_bytes) {
       virtual_bytes_ += additional_virtual_bytes;

tools/deep_memory_profiler/dmprof.py

@@ -736,8 +736,8 @@ class Dump(object):
         lambda n: self._lines[n] != 'GLOBAL_STATS:\n')
 
     global_stat_names = [
-        'total', 'file-exec', 'file-nonexec', 'anonymous', 'stack', 'other',
-        'nonprofiled-absent', 'nonprofiled-anonymous',
+        'total', 'absent', 'file-exec', 'file-nonexec', 'anonymous', 'stack',
+        'other', 'nonprofiled-absent', 'nonprofiled-anonymous',
         'nonprofiled-file-exec', 'nonprofiled-file-nonexec',
         'nonprofiled-stack', 'nonprofiled-other',
         'profiled-mmap', 'profiled-malloc']
@@ -996,6 +996,7 @@ class PolicyCommands(Command):
     for key, value in {
         'total': 'total_committed',
         'filemapped': 'file_committed',
+        'absent': 'absent_committed',
         'file-exec': 'file-exec_committed',
         'file-nonexec': 'file-nonexec_committed',
         'anonymous': 'anonymous_committed',

tools/deep_memory_profiler/policy.l0.json

@@ -20,6 +20,7 @@
     "UNTIL_HERE_FOR_TOTAL",
     "total-exclude-profiler",
     "total",
+    "absent",
     "anonymous",
     "file-exec",
     "file-nonexec",
@@ -125,6 +126,11 @@
       "stacktrace": "optional",
       "allocator": "optional"
     },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
     {
       "name": "anonymous",
       "stacktrace": "optional",

tools/deep_memory_profiler/policy.l1.json

@@ -27,6 +27,7 @@
     "UNTIL_HERE_FOR_TOTAL",
     "total-exclude-profiler",
     "total",
+    "absent",
     "anonymous",
     "file-exec",
     "file-nonexec",
@@ -167,6 +168,11 @@
       "stacktrace": "optional",
       "allocator": "optional"
     },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
     {
       "name": "anonymous",
       "stacktrace": "optional",

tools/deep_memory_profiler/policy.l2.json

@@ -54,6 +54,7 @@
     "UNTIL_HERE_FOR_TOTAL",
     "total-exclude-profiler",
     "total",
+    "absent",
     "anonymous",
     "file-exec",
     "file-nonexec",
@@ -439,6 +440,11 @@
       "stacktrace": "optional",
       "allocator": "optional"
     },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
     {
       "name": "anonymous",
       "stacktrace": "optional",

tools/deep_memory_profiler/policy.t0.json

@@ -26,6 +26,7 @@
     "UNTIL_HERE_FOR_TOTAL",
     "total-exclude-profiler",
     "total",
+    "absent",
     "anonymous",
     "file-exec",
     "file-nonexec",
@@ -168,6 +169,11 @@
       "stacktrace": "optional",
       "allocator": "optional"
     },
+    {
+      "name": "absent",
+      "stacktrace": "optional",
+      "allocator": "optional"
+    },
     {
       "name": "anonymous",
       "stacktrace": "optional",