Normalized APK Size: Ignore size of apk signature block

Also updates SuperSize to track the signature block size as metadata
rather than a symbol so that the symbol diffs match the normalized apk
size.

Bug: 1130754
Change-Id: I759616f54ba8c78f0a193aae075dc8ae9ba6190c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2431764
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Samuel Huang <huangs@chromium.org>
Cr-Commit-Position: refs/heads/master@{#810643}

build/android/resource_sizes.py

@@ -106,6 +106,30 @@ def _ReadZipInfoExtraFieldLength(zip_file, zip_info):
   return struct.unpack('<H', zip_file.fp.read(2))[0]
 
 
+def _MeasureApkSignatureBlock(zip_file):
+  """Measures the size of the v2 / v3 signing block.
+
+  Refer to: https://source.android.com/security/apksigning/v2
+  """
+  # Seek to "end of central directory" struct.
+  eocd_offset_from_end = -22 - len(zip_file.comment)
+  zip_file.fp.seek(eocd_offset_from_end, os.SEEK_END)
+  assert zip_file.fp.read(4) == b'PK\005\006', (
+      'failed to find end-of-central-directory')
+
+  # Read out the "start of central directory" offset.
+  zip_file.fp.seek(eocd_offset_from_end + 16, os.SEEK_END)
+  start_of_central_directory = struct.unpack('<I', zip_file.fp.read(4))[0]
+
+  # Compute the offset after the last zip entry.
+  last_info = zip_file.infolist()[-1]
+  last_header_size = (30 + len(last_info.filename) +
+                      _ReadZipInfoExtraFieldLength(zip_file, last_info))
+  end_of_last_file = (last_info.header_offset + last_header_size +
+                      last_info.compress_size)
+  return start_of_central_directory - end_of_last_file
+
+
 def _RunReadelf(so_path, options, tool_prefix=''):
   return cmd_helper.GetCmdOutput(
       [tool_prefix + 'readelf'] + options + [so_path])
@@ -322,6 +346,7 @@ def _DoApkAnalysis(apk_filename, apks_path, tool_prefix, out_dir, report_func):
     # Happens when python aligns entries in apkbuilder.py, but does not
     # exist when using Android's zipalign. E.g. for bundle .apks files.
     zipalign_overhead += sum(len(i.extra) for i in apk_contents)
+    signing_block_size = _MeasureApkSignatureBlock(apk)
 
   sdk_version, skip_extract_lib = _ParseManifestAttributes(apk_filename)
 
@@ -490,9 +515,13 @@ def _DoApkAnalysis(apk_filename, apks_path, tool_prefix, out_dir, report_func):
   normalized_apk_size += java_code.ComputeUncompressedSize()
   # Don't include zipalign overhead in normalized size, since it effectively
   # causes size changes files that proceed aligned files to be rounded.
-  # For APKs where classes.dex directly proceeds libchrome.so, this causes
+  # For APKs where classes.dex directly proceeds libchrome.so (the normal case),
-  # small dex size changes to disappear into libchrome.so alignment.
+  # this causes small dex size changes to disappear into libchrome.so alignment.
   normalized_apk_size -= zipalign_overhead
+  # Don't include the size of the apk's signing block because it can fluctuate
+  # by up to 4kb (from my non-scientific observations), presumably based on hash
+  # sizes.
+  normalized_apk_size -= signing_block_size
 
   # Unaligned size should be ~= uncompressed size or something is wrong.
   # As of now, padding_fraction ~= .007

docs/speed/binary_size/metrics.md

@@ -37,10 +37,15 @@ For Googlers, more information available at [go/chrome-apk-size](https://goto.go
  * Computed as:
    * The size of an APK
    * With all native code as the sum of section sizes (except .bss), uncompressed.
+     * Why: Removes effects of ELF section alignment.
    * With all dex code as if it were stored uncompressed.
+     * Why: Dex is stored uncompressed on newer Android versions.
    * With all zipalign padding removed.
+     * Why: Removes effects of file alignment (esp. relevant because native libraries are 4k-aligned).
+   * With size of apk signature block removed.
+     * Why: Size fluctuates by several KB based on how hash values turn out.
    * With all translations as if they were not missing (estimates size of missing translations based on size of english strings).
-     * Without translation-normalization, translation dumps cause jumps.
+     * Why: Without translation-normalization, translation dumps cause jumps.
      * Translation-normalization applies only to apks (not to Android App Bundles).
 
 ### Native Code Size Metrics

tools/binary_size/libsupersize/archive.py

@@ -1279,11 +1279,13 @@ def _ParseApkOtherSymbols(section_ranges, apk_path, apk_so_path,
               source_path=source_path,
               full_name=resource_filename))  # Full name must disambiguate
 
-  if signing_block_size > 0:
+  # Store zipalign overhead and signing block size as metadata rather than an
-    signing_symbol = models.Symbol(models.SECTION_OTHER,
+  # "Overhead:" symbol because they fluctuate in size, and would be a source of
-                                   signing_block_size,
+  # noise in symbol diffs if included as symbols (http://crbug.com/1130754).
-                                   full_name='APK Signature Block')
+  # Might be even better if we had an option in Tiger Viewer to ignore certain
-    apk_symbols.append(signing_symbol)
+  # symbols, but taking this as a short-cut for now.
+  metadata[models.METADATA_ZIPALIGN_OVERHEAD] = zipalign_total
+  metadata[models.METADATA_SIGNING_BLOCK_SIZE] = signing_block_size
 
   # Overhead includes:
   #  * Size of all local zip headers (minus zipalign padding).
@@ -1294,9 +1296,6 @@ def _ParseApkOtherSymbols(section_ranges, apk_path, apk_so_path,
   zip_overhead_symbol = models.Symbol(
       models.SECTION_OTHER, overhead_size, full_name='Overhead: APK file')
   apk_symbols.append(zip_overhead_symbol)
-  # Store as metadata rather than an Overhead: symbol so that the sum of symbols
-  # matches normalized apk size.
-  metadata[models.METADATA_ZIPALIGN_OVERHEAD] = zipalign_total
   _ExtendSectionRange(section_ranges, models.SECTION_OTHER,
                       sum(s.size for s in apk_symbols))
   return dex_size, apk_symbols

tools/binary_size/libsupersize/models.py

@@ -53,6 +53,7 @@ BUILD_CONFIG_KEYS = (
 METADATA_APK_FILENAME = 'apk_file_name'  # Path relative to output_directory.
 METADATA_APK_SIZE = 'apk_size'  # File size of apk in bytes.
 METADATA_ZIPALIGN_OVERHEAD = 'zipalign_padding'  # Overhead from zipalign.
+METADATA_SIGNING_BLOCK_SIZE = 'apk_signature_block_size'  # Size in bytes.
 METADATA_MAP_FILENAME = 'map_file_name'  # Path relative to output_directory.
 METADATA_ELF_ARCHITECTURE = 'elf_arch'  # "Machine" field from readelf -h
 METADATA_ELF_FILENAME = 'elf_file_name'  # Path relative to output_directory.

tools/binary_size/libsupersize/testdata/Archive_Apk.golden

@@ -1,4 +1,5 @@
 apk_file_name=test.apk
+apk_signature_block_size=0
 apk_size=147858911
 elf_arch=arm
 elf_build_id=WhatAnAmazingBuildId

tools/binary_size/libsupersize/testdata/Archive_MinimalApks.golden

@@ -1,4 +1,5 @@
 apk_file_name=Bundle.minimal.apks
+apk_signature_block_size=0
 apk_size-vr=20
 apk_size=147858911
 elf_arch=arm