Android: Templates and targets for building system image apks

system_image_stub_apk() can build a -Stub.apk for use with compressed
system image apk files (.apk.gz files).

system_image_apks() can create a single fused .apk, or a zip of apk
splits suitable for system image.

"Suitable" means:
 * Does not break out locales into separate splits
 * Compresses .dex files when fuse_apk=true

Bug: 1324820
Change-Id: Ib2843c7f9e1eb830ccec9a8dd2fba78b932c287f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3645630
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Samuel Huang <huangs@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1003776}

PRESUBMIT.py

@@ -1115,6 +1115,7 @@ _GENERIC_PYDEPS_FILES = [
     'build/android/gyp/prepare_resources.pydeps',
     'build/android/gyp/process_native_prebuilt.pydeps',
     'build/android/gyp/proguard.pydeps',
+    'build/android/gyp/system_image_apks.pydeps',
     'build/android/gyp/trace_event_bytecode_rewriter.pydeps',
     'build/android/gyp/turbine.pydeps',
     'build/android/gyp/unused_resources.pydeps',

build/android/gyp/compile_resources.py

@@ -12,6 +12,7 @@ final R.java class for all resource packages the APK depends on.
 This will crunch images with aapt2.
 """
 
+import argparse
 import collections
 import contextlib
 import filecmp
@@ -51,7 +52,26 @@ def _ParseArgs(args):
   Returns:
     An options object as from argparse.ArgumentParser.parse_args()
   """
-  parser, input_opts, output_opts = resource_utils.ResourceArgsParser()
+  parser = argparse.ArgumentParser(description=__doc__)
+
+  input_opts = parser.add_argument_group('Input options')
+  output_opts = parser.add_argument_group('Output options')
+
+  input_opts.add_argument('--include-resources',
+                          action='append',
+                          required=True,
+                          help='Paths to arsc resource files used to link '
+                          'against. Can be specified multiple times.')
+
+  input_opts.add_argument(
+      '--dependencies-res-zips',
+      default=[],
+      help='Resources zip archives from dependents. Required to '
+      'resolve @type/foo references into dependent libraries.')
+
+  input_opts.add_argument(
+      '--extra-res-packages',
+      help='Additional package names to generate R.java files for.')
 
   input_opts.add_argument(
       '--aapt2-path', required=True, help='Path to the Android aapt2 tool.')
@@ -176,6 +196,21 @@ def _ParseArgs(args):
       action='store_true',
       help='Whether to strip xml namespaces from processed xml resources.')
 
+  input_opts.add_argument(
+      '--is-bundle-module',
+      action='store_true',
+      help='Whether resources are being generated for a bundle module.')
+
+  input_opts.add_argument(
+      '--uses-split',
+      help='Value to set uses-split to in the AndroidManifest.xml.')
+
+  input_opts.add_argument(
+      '--extra-verification-manifest',
+      help='Path to AndroidManifest.xml which should be merged into base '
+      'manifest when performing verification.')
+
+  build_utils.AddDepfileOption(output_opts)
   output_opts.add_argument('--arsc-path', help='Apk output for arsc format.')
   output_opts.add_argument('--proto-path', help='Apk output for proto format.')
 
@@ -184,7 +219,6 @@ def _ParseArgs(args):
 
   output_opts.add_argument(
       '--srcjar-out',
-      required=True,
       help='Path to srcjar to contain generated R.java.')
 
   output_opts.add_argument('--r-text-out',
@@ -200,25 +234,14 @@ def _ParseArgs(args):
   output_opts.add_argument(
       '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.')
 
-  input_opts.add_argument(
-      '--is-bundle-module',
-      action='store_true',
-      help='Whether resources are being generated for a bundle module.')
-
-  input_opts.add_argument(
-      '--uses-split',
-      help='Value to set uses-split to in the AndroidManifest.xml.')
-
-  input_opts.add_argument(
-      '--extra-verification-manifest',
-      help='Path to AndroidManifest.xml which should be merged into base '
-      'manifest when performing verification.')
-
   diff_utils.AddCommandLineFlags(parser)
   options = parser.parse_args(args)
 
-  resource_utils.HandleCommonOptions(options)
+  options.include_resources = build_utils.ParseGnList(options.include_resources)
-
+  options.dependencies_res_zips = build_utils.ParseGnList(
+      options.dependencies_res_zips)
+  options.extra_res_packages = build_utils.ParseGnList(
+      options.extra_res_packages)
   options.locale_allowlist = build_utils.ParseGnList(options.locale_allowlist)
   options.shared_resources_allowlist_locales = build_utils.ParseGnList(
       options.shared_resources_allowlist_locales)
@@ -830,9 +853,10 @@ def _PackageApk(options, build):
   link_proc = subprocess.Popen(link_command)
 
   # Create .res.info file in parallel.
-  _CreateResourceInfoFile(path_info, build.info_path,
+  if options.info_path:
-                          options.dependencies_res_zips)
+    logging.debug('Creating .res.info file')
-  logging.debug('Created .res.info file')
+    _CreateResourceInfoFile(path_info, build.info_path,
+                            options.dependencies_res_zips)
 
   exit_code = link_proc.wait()
   assert exit_code == 0, f'aapt2 link cmd failed with {exit_code=}'
@@ -1010,18 +1034,20 @@ def main(args):
       # will be created in the base module.
       apk_package_name = None
 
-    logging.debug('Creating R.srcjar')
+    if options.srcjar_out:
-    resource_utils.CreateRJavaFiles(
+      logging.debug('Creating R.srcjar')
-        build.srcjar_dir, apk_package_name, build.r_txt_path,
+      resource_utils.CreateRJavaFiles(
-        options.extra_res_packages, rjava_build_options, options.srcjar_out,
+          build.srcjar_dir, apk_package_name, build.r_txt_path,
-        custom_root_package_name, grandparent_custom_package_name,
+          options.extra_res_packages, rjava_build_options, options.srcjar_out,
-        options.extra_main_r_text_files)
+          custom_root_package_name, grandparent_custom_package_name,
-    build_utils.ZipDir(build.srcjar_path, build.srcjar_dir)
+          options.extra_main_r_text_files)
+      build_utils.ZipDir(build.srcjar_path, build.srcjar_dir)
 
     logging.debug('Copying outputs')
     _WriteOutputs(options, build)
 
   if options.depfile:
+    assert options.srcjar_out, 'Update first output below and remove assert.'
     depfile_deps = (options.dependencies_res_zips +
                     options.dependencies_res_zip_overlays +
                     options.extra_main_r_text_files + options.include_resources)

build/android/gyp/system_image_apks.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Generates APKs for use on system images."""
+
+import argparse
+import os
+import pathlib
+import tempfile
+import shutil
+import sys
+import zipfile
+
+_DIR_SOURCE_ROOT = str(pathlib.Path(__file__).parents[2])
+sys.path.append(os.path.join(_DIR_SOURCE_ROOT, 'build', 'android', 'gyp'))
+from util import build_utils
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--input', required=True, help='Input path')
+  parser.add_argument('--output', required=True, help='Output path')
+  parser.add_argument('--bundle-wrapper', help='APK operations script path')
+  parser.add_argument('--fuse-apk',
+                      help='Create single .apk rather than using apk splits',
+                      action='store_true')
+  args = parser.parse_args()
+
+  if not args.bundle_wrapper:
+    shutil.copyfile(args.input, args.output)
+    return
+
+  with tempfile.NamedTemporaryFile(suffix='.apks') as tmp_file:
+    cmd = [
+        args.bundle_wrapper, 'build-bundle-apks', '--output-apks', tmp_file.name
+    ]
+    cmd += ['--build-mode', 'system' if args.fuse_apk else 'system_apks']
+
+    # Creates a .apks zip file that contains the system image APK(s).
+    build_utils.CheckOutput(cmd)
+
+    if args.fuse_apk:
+      with zipfile.ZipFile(tmp_file.name) as z:
+        pathlib.Path(args.output).write_bytes(z.read('system/system.apk'))
+      return
+
+    # Rename .apk files and remove toc.pb to make it clear that system apks
+    # should not be installed via bundletool.
+    with zipfile.ZipFile(tmp_file.name) as z_input, \
+        zipfile.ZipFile(args.output, 'w') as z_output:
+      for info in z_input.infolist():
+        if info.filename.endswith('.apk'):
+          data = z_input.read(info)
+          info.filename = (info.filename.replace('splits/',
+                                                 '').replace('-master', ''))
+          z_output.writestr(info, data)
+
+
+if __name__ == '__main__':
+  sys.exit(main())

build/android/gyp/system_image_apks.pydeps

@@ -0,0 +1,6 @@
+# Generated by running:
+#   build/print_python_deps.py --root build/android/gyp --output build/android/gyp/system_image_apks.pydeps build/android/gyp/system_image_apks.py
+../../gn_helpers.py
+system_image_apks.py
+util/__init__.py
+util/build_utils.py

build/android/gyp/util/resource_utils.py

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import argparse
 import collections
 import contextlib
 import itertools
@@ -951,60 +950,6 @@ def BuildContext(temp_dir=None, keep_files=False):
       context.Close()
 
 
-def ResourceArgsParser():
-  """Create an argparse.ArgumentParser instance with common argument groups.
-
-  Returns:
-    A tuple of (parser, in_group, out_group) corresponding to the parser
-    instance, and the input and output argument groups for it, respectively.
-  """
-  parser = argparse.ArgumentParser(description=__doc__)
-
-  input_opts = parser.add_argument_group('Input options')
-  output_opts = parser.add_argument_group('Output options')
-
-  build_utils.AddDepfileOption(output_opts)
-
-  input_opts.add_argument('--include-resources', required=True, action="append",
-                        help='Paths to arsc resource files used to link '
-                             'against. Can be specified multiple times.')
-
-  input_opts.add_argument('--dependencies-res-zips', required=True,
-                    help='Resources zip archives from dependents. Required to '
-                         'resolve @type/foo references into dependent '
-                         'libraries.')
-
-  input_opts.add_argument(
-      '--extra-res-packages',
-      help='Additional package names to generate R.java files for.')
-
-  return (parser, input_opts, output_opts)
-
-
-def HandleCommonOptions(options):
-  """Handle common command-line options after parsing.
-
-  Args:
-    options: the result of parse_args() on the parser returned by
-        ResourceArgsParser(). This function updates a few common fields.
-  """
-  options.include_resources = [build_utils.ParseGnList(r) for r in
-                               options.include_resources]
-  # Flatten list of include resources list to make it easier to use.
-  options.include_resources = [r for resources in options.include_resources
-                               for r in resources]
-
-  options.dependencies_res_zips = (
-      build_utils.ParseGnList(options.dependencies_res_zips))
-
-  # Don't use [] as default value since some script explicitly pass "".
-  if options.extra_res_packages:
-    options.extra_res_packages = (
-        build_utils.ParseGnList(options.extra_res_packages))
-  else:
-    options.extra_res_packages = []
-
-
 def ParseAndroidResourceStringsFromXml(xml_data):
   """Parse and Android xml resource file and extract strings from it.
 

build/android/gyp/write_build_config.py

@@ -1763,6 +1763,7 @@ def main(argv):
       if c['is_base_module']:
         assert 'base_module_config' not in deps_info, (
             'Must have exactly 1 base module!')
+        deps_info['package_name'] = c['package_name']
         deps_info['base_module_config'] = c['path']
         # Use the base module's android manifest for linting.
         deps_info['lint_android_manifest'] = c['android_manifest']

build/config/android/internal_rules.gni

@@ -2437,13 +2437,13 @@ if (enable_java_templates) {
 
     action_with_pydeps(target_name) {
       script = _script
-      depfile = "$target_gen_dir/${target_name}.d"
+      _depfile = "$target_gen_dir/${target_name}.d"
       inputs = _inputs
       outputs = _outputs
       deps = _deps
       args = _args + [
                "--depfile",
-               rebase_path(depfile, root_build_dir),
+               rebase_path(_depfile, root_build_dir),
              ]
     }
   }
@@ -2679,9 +2679,18 @@ if (enable_java_templates) {
     _apksigner = "$android_sdk_build_tools/lib/apksigner.jar"
     _zipalign = "$android_sdk_build_tools/zipalign"
 
+    _keystore_path = android_keystore_path
+    _keystore_name = android_keystore_name
+    _keystore_password = android_keystore_password
+
+    if (defined(invoker.keystore_path)) {
+      _keystore_path = invoker.keystore_path
+      _keystore_name = invoker.keystore_name
+      _keystore_password = invoker.keystore_password
+    }
+
     _inputs = [
-      invoker.build_config,
+      _keystore_path,
-      invoker.keystore_path,
       invoker.packaged_resources_path,
       _apksigner,
       _zipalign,
@@ -2694,28 +2703,30 @@ if (enable_java_templates) {
         rebase_path(invoker.packaged_resources_path, root_build_dir)
     _rebased_packaged_apk_path =
         rebase_path(invoker.output_apk_path, root_build_dir)
-    _rebased_build_config = rebase_path(invoker.build_config, root_build_dir)
     _args = [
       "--resource-apk=$_rebased_compiled_resources_path",
       "--output-apk=$_rebased_packaged_apk_path",
-      "--assets=@FileArg($_rebased_build_config:assets)",
-      "--uncompressed-assets=@FileArg($_rebased_build_config:uncompressed_assets)",
       "--apksigner-jar",
       rebase_path(_apksigner, root_build_dir),
       "--zipalign-path",
       rebase_path(_zipalign, root_build_dir),
       "--key-path",
-      rebase_path(invoker.keystore_path, root_build_dir),
+      rebase_path(_keystore_path, root_build_dir),
       "--key-name",
-      invoker.keystore_name,
+      _keystore_name,
       "--key-passwd",
-      invoker.keystore_password,
+      _keystore_password,
       "--min-sdk-version=${invoker.min_sdk_version}",
-
-      # TODO(mlopatkin) We are relying on the fact that build_config is an APK
-      # build_config.
-      "--java-resources=@FileArg($_rebased_build_config:java_resources_jars)",
     ]
+    if (defined(invoker.build_config)) {
+      _inputs += [ invoker.build_config ]
+      _rebased_build_config = rebase_path(invoker.build_config, root_build_dir)
+      _args += [
+        "--assets=@FileArg($_rebased_build_config:assets)",
+        "--uncompressed-assets=@FileArg($_rebased_build_config:uncompressed_assets)",
+        "--java-resources=@FileArg($_rebased_build_config:java_resources_jars)",
+      ]
+    }
     if (is_official_build) {
       _args += [ "--best-compression" ]
     }
@@ -2794,10 +2805,10 @@ if (enable_java_templates) {
         _failure_file =
             "$expectations_failure_dir/" +
             string_replace(invoker.expected_libs_and_assets, "/", "_")
-        inputs = [
+        inputs = [ invoker.expected_libs_and_assets ]
-          invoker.build_config,
+        if (defined(invoker.build_config)) {
-          invoker.expected_libs_and_assets,
+          inputs += [ invoker.build_config ]
-        ]
+        }
         deps = [ invoker.build_config_dep ]
         outputs = [
           _actual_file,

build/config/android/rules.gni

@@ -2194,7 +2194,7 @@ if (enable_java_templates) {
   #     with this file as the base.
   template("android_apk_or_module") {
     forward_variables_from(invoker, TESTONLY_AND_VISIBILITY)
-    assert(defined(invoker.android_manifest))
+    _template_name = target_name
     _base_path = "$target_out_dir/$target_name/$target_name"
     _build_config = "$target_gen_dir/$target_name.build_config.json"
     _build_config_target = "$target_name$build_config_target_suffix"
@@ -2208,8 +2208,6 @@ if (enable_java_templates) {
       _target_sdk_version = invoker.target_sdk_version
     }
 
-    _template_name = target_name
-
     _is_bundle_module =
         defined(invoker.is_bundle_module) && invoker.is_bundle_module
     if (_is_bundle_module) {
@@ -3141,16 +3139,6 @@ if (enable_java_templates) {
         }
       }
 
-      _keystore_path = android_keystore_path
-      _keystore_name = android_keystore_name
-      _keystore_password = android_keystore_password
-
-      if (defined(invoker.keystore_path)) {
-        _keystore_path = invoker.keystore_path
-        _keystore_name = invoker.keystore_name
-        _keystore_password = invoker.keystore_password
-      }
-
       if (_incremental_apk) {
         _incremental_compiled_resources_path = "${_base_path}_incremental.ap_"
         _incremental_compile_resources_target_name =
@@ -3193,6 +3181,9 @@ if (enable_java_templates) {
                                [
                                  "expected_libs_and_assets",
                                  "expected_libs_and_assets_base",
+                                 "keystore_name",
+                                 "keystore_path",
+                                 "keystore_password",
                                  "native_lib_placeholders",
                                  "secondary_abi_loadable_modules",
                                  "secondary_native_lib_placeholders",
@@ -3208,9 +3199,6 @@ if (enable_java_templates) {
         }
 
         build_config = _build_config
-        keystore_name = _keystore_name
-        keystore_path = _keystore_path
-        keystore_password = _keystore_password
         min_sdk_version = _min_sdk_version
         uncompress_shared_libraries = _uncompress_shared_libraries
 

build/config/android/system_image.gni

@@ -0,0 +1,124 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/config/android/rules.gni")
+
+# Creates a stub .apk suitable for use with compressed system APKs.
+#
+# Variables:
+#   package_name: Package name to use for the stub.
+#   package_name_from_target: Use the package name from this apk/bundle target.
+#   stub_output: Path to output stub apk (default: do not create a stub).
+#
+# package_name and package_name_from_target are mutually exclusive.
+template("system_image_stub_apk") {
+  # Android requires stubs end with -Stub.apk.
+  assert(filter_exclude([ invoker.stub_output ], [ "*-Stub.apk" ]) == [],
+         "stub_output \"${invoker.stub_output}\" must end with \"-Stub.apk\"")
+
+  _resource_apk_path = "${target_out_dir}/$target_name.ap_"
+  _resource_apk_target_name = "${target_name}__compile_resources"
+  action_with_pydeps(_resource_apk_target_name) {
+    _manifest_path = "//build/android/AndroidManifest.xml"
+    script = "//build/android/gyp/compile_resources.py"
+    inputs = [
+      _manifest_path,
+      android_sdk_jar,
+    ]
+    outputs = [ _resource_apk_path ]
+    args = [
+      "--aapt2-path",
+      rebase_path(android_sdk_tools_bundle_aapt2, root_build_dir),
+      "--min-sdk-version=$default_min_sdk_version",
+      "--target-sdk-version=$default_min_sdk_version",
+      "--android-manifest",
+      rebase_path(_manifest_path, root_build_dir),
+      "--arsc-path",
+      rebase_path(_resource_apk_path, root_build_dir),
+    ]
+    if (defined(invoker.package_name)) {
+      _package_name = invoker.package_name
+    } else {
+      _target = invoker.package_name_from_target
+      deps = [ "${_target}$build_config_target_suffix" ]
+      _build_config = get_label_info(_target, "target_gen_dir") + "/" +
+                      get_label_info(_target, "name") + ".build_config.json"
+      inputs += [ _build_config ]
+      _rebased_build_config = rebase_path(_build_config, root_build_dir)
+      _package_name = "@FileArg($_rebased_build_config:deps_info:package_name)"
+    }
+    args += [
+      "--rename-manifest-package=$_package_name",
+      "--arsc-package-name=$_package_name",
+      "--include-resources",
+      rebase_path(android_sdk_jar, root_build_dir),
+    ]
+  }
+
+  package_apk(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "keystore_name",
+                             "keystore_path",
+                             "keystore_password",
+                           ])
+    min_sdk_version = default_min_sdk_version
+    deps = [ ":$_resource_apk_target_name" ]
+
+    packaged_resources_path = _resource_apk_path
+    output_apk_path = invoker.stub_output
+  }
+}
+
+# Generates artifacts for system APKs.
+#
+# Variables:
+#   apk_or_bundle_target: Target that creates input bundle or apk.
+#   input_apk_or_bundle: Path to input .apk or .aab.
+#   output: Path to the output system .apk or .zip.
+#   fuse_apk: Fuse all apk splits into a single .apk (default: false).
+#   stub_output: Path to output stub apk (default: do not create a stub).
+#
+template("system_image_apks") {
+  if (defined(invoker.stub_output)) {
+    _stub_apk_target_name = "${target_name}__stub"
+    system_image_stub_apk(_stub_apk_target_name) {
+      package_name_from_target = invoker.apk_or_bundle_target
+      stub_output = invoker.stub_output
+    }
+  }
+
+  action_with_pydeps(target_name) {
+    script = "//build/android/gyp/system_image_apks.py"
+    deps = [ invoker.apk_or_bundle_target ]
+    inputs = [ invoker.input_apk_or_bundle ]
+    if (defined(invoker.stub_output)) {
+      public_deps = [ ":$_stub_apk_target_name" ]
+    }
+    outputs = [ invoker.output ]
+    args = [
+      "--input",
+      rebase_path(invoker.input_apk_or_bundle, root_out_dir),
+      "--output",
+      rebase_path(invoker.output, root_out_dir),
+    ]
+
+    _is_bundle =
+        filter_exclude([ invoker.input_apk_or_bundle ], [ "*.aab" ]) == []
+
+    if (_is_bundle) {
+      _wrapper_path = "$root_out_dir/bin/" +
+                      get_label_info(invoker.apk_or_bundle_target, "name")
+      args += [
+        "--bundle-wrapper",
+        rebase_path(_wrapper_path, root_out_dir),
+      ]
+      inputs += [ _wrapper_path ]
+      deps += [ "//build/android:apk_operations_py" ]
+      if (defined(invoker.fuse_apk) && invoker.fuse_apk) {
+        args += [ "--fuse-apk" ]
+      }
+    }
+  }
+}

chrome/android/BUILD.gn

@@ -5,6 +5,7 @@
 import("//build/android/resource_sizes.gni")
 import("//build/config/android/config.gni")
 import("//build/config/android/rules.gni")
+import("//build/config/android/system_image.gni")
 import("//build/config/python.gni")
 import("//build/util/process_version.gni")
 import("//build/util/version.gni")
@@ -2862,6 +2863,12 @@ if (android_64bit_target_cpu && skip_secondary_abi_for_cq) {
       expected_libs_and_assets = "expectations/trichrome_library_apk.$target_cpu.libs_and_assets.expected"
     }
   }
+
+  # Can be used to install compressed apks on system images.
+  system_image_stub_apk("trichrome_library_system_stub_apk") {
+    package_name = chrome_public_manifest_package
+    stub_output = "$root_out_dir/apks/TrichromeLibrary-Stub.apk"
+  }
 }
 
 if (android_64bit_target_cpu) {
@@ -3696,6 +3703,22 @@ if (android_64bit_target_cpu && skip_secondary_abi_for_cq) {
     }
   }
 
+  # Creates .zip of .apk splits suitable for the Android system image.
+  system_image_apks("trichrome_chrome_system_zip") {
+    apk_or_bundle_target = ":trichrome_chrome_bundle"
+    input_apk_or_bundle = "$root_out_dir/apks/TrichromeChrome.aab"
+    output = "$root_out_dir/apks/TrichromeChromeSystem.zip"
+    stub_output = "$root_out_dir/apks/TrichromeChrome-Stub.apk"
+  }
+
+  # Combines all splits into a single .apk for the Android system image.
+  system_image_apks("trichrome_chrome_system_apk") {
+    apk_or_bundle_target = ":trichrome_chrome_bundle"
+    input_apk_or_bundle = "$root_out_dir/apks/TrichromeChrome.aab"
+    output = "$root_out_dir/apks/TrichromeChromeSystem.apk"
+    fuse_apk = true
+  }
+
   if (is_official_build) {
     _trichrome_library_basename = "TrichromeLibrary.apk"
     _trichrome_chrome_basename = "TrichromeChrome.minimal.apks"