remove action_remote.py
action_remote.py was used to make action remote executable if allow_remote=true. It is no longer needed for Siso, as we can configure remote execution in siso config. action template is the only place to modify build graph by `use_siso` value, so we can remove //build/toolchain/use_siso_value.py if we remove these codes. Bug: 398130236 Change-Id: I1106de842c2e6d95bd249666e388e50c3cc11c96 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6289606 Reviewed-by: Takuto Ikuta <tikuta@chromium.org> Auto-Submit: Fumitoshi Ukai <ukai@google.com> Reviewed-by: Daniel Cheng <dcheng@chromium.org> Commit-Queue: Fumitoshi Ukai <ukai@google.com> Reviewed-by: Junji Watanabe <jwata@google.com> Reviewed-by: Philipp Wollermann <philwo@google.com> Cr-Commit-Position: refs/heads/main@{#1424947}
This commit is contained in:
Fumitoshi Ukai
committed by
Chromium LUCI CQ
Chromium LUCI CQ
parent
f9e262778b
commit
7b86f2ed4d
build
mojo/public/tools/bindings
@ -765,130 +765,3 @@ if (is_component_build) {
|
||||
set_defaults("component") {
|
||||
configs = default_component_configs
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# ACTION OVERRIDE
|
||||
# =============================================================================
|
||||
#
|
||||
# We override gn action() to support remote execution using rewrapper. The
|
||||
# invoker should set allow_remote to true if remote execution is desired.
|
||||
#
|
||||
# As remote execution requires inputs to be made more explicit than is normally
|
||||
# expected with gn, you may find that setting allow_remote to true will result
|
||||
# in many missing file errors. In most cases, this should be resolved by
|
||||
# explicitly declaring these inputs/sources.
|
||||
#
|
||||
# However, it may be impractical to determine these inputs in gn. For such
|
||||
# cases, the invoker can specify a custom input processor, which are currently
|
||||
# defined and implemented in //build/util/action_remote.py. The appropriate
|
||||
# value should be set using the custom_processor arg.
|
||||
|
||||
# Variables needed by rbe.gni aren't available at the top of this file.
|
||||
import("//build/toolchain/rbe.gni")
|
||||
import("//build/toolchain/siso.gni")
|
||||
|
||||
# TODO(b/253987456): Add action_foreach support.
|
||||
# TODO(379584977): remove this. no need this for siso native.
|
||||
foreach(_target_type, [ "action" ]) {
|
||||
template(_target_type) {
|
||||
forward_variables_from(invoker, TESTONLY_AND_VISIBILITY)
|
||||
action("${target_name}") {
|
||||
forward_variables_from(invoker,
|
||||
[
|
||||
"args",
|
||||
"assert_no_deps",
|
||||
"check_includes",
|
||||
"configs",
|
||||
"data_deps",
|
||||
"data",
|
||||
"depfile",
|
||||
"deps",
|
||||
"metadata",
|
||||
"outputs",
|
||||
"pool",
|
||||
"script",
|
||||
"public_configs",
|
||||
"public_deps",
|
||||
"response_file_contents",
|
||||
"sources",
|
||||
"write_runtime_deps",
|
||||
])
|
||||
allow_remote = false
|
||||
if (defined(invoker.allow_remote)) {
|
||||
allow_remote = invoker.allow_remote
|
||||
}
|
||||
|
||||
# If remote execution is desired, only run remotely when use_remoteexec
|
||||
# is enabled, and the environment is not nacl.
|
||||
# Siso doesn't use action_remote.py wrapper because it sends requests to
|
||||
# Reproxy directly without actions_remote.py/rewrapper.
|
||||
# TODO(b/259381924): Investigate enabling in nacl config.
|
||||
if (allow_remote && use_reclient && !is_nacl && !use_siso) {
|
||||
pool = "//build/toolchain:remote_action_pool($default_toolchain)"
|
||||
script = "//build/util/action_remote.py"
|
||||
inputs = [ invoker.script ]
|
||||
|
||||
re_inputs = [ rebase_path(invoker.script, rbe_exec_root) ]
|
||||
if (defined(invoker.inputs)) {
|
||||
foreach(input, invoker.inputs) {
|
||||
re_inputs += [ rebase_path(input, rbe_exec_root) ]
|
||||
inputs += [ input ]
|
||||
}
|
||||
}
|
||||
if (defined(invoker.sources)) {
|
||||
foreach(source, invoker.sources) {
|
||||
re_inputs += [ rebase_path(source, rbe_exec_root) ]
|
||||
}
|
||||
}
|
||||
|
||||
re_outputs = []
|
||||
if (defined(invoker.outputs)) {
|
||||
foreach(output, invoker.outputs) {
|
||||
re_outputs += [ rebase_path(output, rbe_exec_root) ]
|
||||
}
|
||||
}
|
||||
|
||||
# Write input/output lists to files as these can grow extremely large.
|
||||
re_inputs_file = "$target_gen_dir/${target_name}__remote_inputs.rsp"
|
||||
write_file(re_inputs_file, re_inputs)
|
||||
inputs += [ re_inputs_file ]
|
||||
re_outputs_file = "$target_gen_dir/${target_name}__remote_outputs.rsp"
|
||||
write_file(re_outputs_file, re_outputs)
|
||||
|
||||
rewrapper_cfg = "$reclient_py_cfg_file"
|
||||
if (defined(invoker.remote_worker)) {
|
||||
remote_worker = invoker.remote_worker
|
||||
assert(remote_worker == "large",
|
||||
"remote_worker = " + remote_worker + " is not supported")
|
||||
rewrapper_cfg = "$reclient_py_large_cfg_file"
|
||||
}
|
||||
|
||||
args = []
|
||||
args += [ "$reclient_bin_dir/rewrapper" ]
|
||||
if (defined(invoker.custom_processor)) {
|
||||
args += [ "--custom_processor=" + invoker.custom_processor ]
|
||||
}
|
||||
|
||||
args += [
|
||||
"--cfg=" + rewrapper_cfg,
|
||||
"--exec_root=$rbe_exec_root",
|
||||
"--input_list_paths=" + rebase_path(re_inputs_file, root_build_dir),
|
||||
"--output_list_paths=" + rebase_path(re_outputs_file, root_build_dir),
|
||||
"python3",
|
||||
rebase_path(invoker.script, root_build_dir),
|
||||
]
|
||||
|
||||
if (defined(invoker.args)) {
|
||||
args += invoker.args
|
||||
}
|
||||
} else {
|
||||
forward_variables_from(invoker, [ "inputs" ])
|
||||
not_needed(invoker,
|
||||
[
|
||||
"custom_processor",
|
||||
"remote_worker",
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
# found in the LICENSE file.
|
||||
|
||||
import("//build/config/apple/mobile_config.gni")
|
||||
import("//build/toolchain/rbe.gni")
|
||||
import("//build_overrides/build.gni")
|
||||
|
||||
# Constants corresponding to the bundle type identifiers use application,
|
||||
|
||||
@ -1,155 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2022 The Chromium Authors
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Wrapper script to run action remotely through rewrapper with gn.
|
||||
|
||||
Also includes Chromium-specific input processors which don't make sense to
|
||||
be reclient inbuilt input processors."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from enum import Enum
|
||||
|
||||
_THIS_DIR = os.path.realpath(os.path.dirname(__file__))
|
||||
_SRC_DIR = os.path.dirname(os.path.dirname(_THIS_DIR))
|
||||
_MOJOM_DIR = os.path.join(_SRC_DIR, 'mojo', 'public', 'tools', 'mojom')
|
||||
|
||||
|
||||
class CustomProcessor(Enum):
|
||||
mojom_parser = 'mojom_parser'
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
def _normalize_path(path):
|
||||
# Always use posix-style directory separators as GN does it.
|
||||
return os.path.normpath(path).replace("\\", "/")
|
||||
|
||||
|
||||
def _process_build_metadata_json(bm_file, input_roots, output_root,
|
||||
output_files, processed_inputs):
|
||||
"""Recursively find mojom_parser inputs from a build_metadata file."""
|
||||
# Import Mojo-specific dep here so non-Mojo remote actions don't need it.
|
||||
if _MOJOM_DIR not in sys.path:
|
||||
sys.path.insert(0, _MOJOM_DIR)
|
||||
from mojom_parser import RebaseAbsolutePath
|
||||
|
||||
if bm_file in processed_inputs:
|
||||
return
|
||||
|
||||
processed_inputs.add(bm_file)
|
||||
|
||||
bm_dir = os.path.dirname(bm_file)
|
||||
|
||||
with open(bm_file) as f:
|
||||
bm = json.load(f)
|
||||
|
||||
# All sources and corresponding module files are inputs.
|
||||
for s in bm["sources"]:
|
||||
src = _normalize_path(os.path.join(bm_dir, s))
|
||||
if src not in processed_inputs and os.path.exists(src):
|
||||
processed_inputs.add(src)
|
||||
src_module = _normalize_path(
|
||||
os.path.join(
|
||||
output_root,
|
||||
RebaseAbsolutePath(os.path.abspath(src), input_roots) + "-module"))
|
||||
if src_module in output_files:
|
||||
continue
|
||||
if src_module not in processed_inputs and os.path.exists(src_module):
|
||||
processed_inputs.add(src_module)
|
||||
|
||||
# Recurse into build_metadata deps.
|
||||
for d in bm["deps"]:
|
||||
dep = _normalize_path(os.path.join(bm_dir, d))
|
||||
_process_build_metadata_json(dep, input_roots, output_root, output_files,
|
||||
processed_inputs)
|
||||
|
||||
|
||||
def _get_mojom_parser_inputs(exec_root, output_files, extra_args):
|
||||
"""Get mojom inputs by walking generated build_metadata files.
|
||||
|
||||
This is less complexity and disk I/O compared to parsing mojom files for
|
||||
imports and finding all imports.
|
||||
|
||||
Start from the root build_metadata file passed to mojom_parser's
|
||||
--check-imports flag.
|
||||
"""
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('--check-imports', dest='check_imports', required=True)
|
||||
argparser.add_argument('--output-root', dest='output_root', required=True)
|
||||
argparser.add_argument('--input-root',
|
||||
default=[],
|
||||
action='append',
|
||||
dest='input_root_paths')
|
||||
mojom_parser_args, _ = argparser.parse_known_args(args=extra_args)
|
||||
|
||||
input_roots = list(map(os.path.abspath, mojom_parser_args.input_root_paths))
|
||||
output_root = os.path.abspath(mojom_parser_args.output_root)
|
||||
processed_inputs = set()
|
||||
_process_build_metadata_json(mojom_parser_args.check_imports, input_roots,
|
||||
output_root, output_files, processed_inputs)
|
||||
|
||||
# Rebase paths onto rewrapper exec root.
|
||||
return map(lambda dep: _normalize_path(os.path.relpath(dep, exec_root)),
|
||||
processed_inputs)
|
||||
|
||||
|
||||
def main():
|
||||
# Set up argparser with some rewrapper flags.
|
||||
argparser = argparse.ArgumentParser(description='rewrapper executor for gn',
|
||||
allow_abbrev=False)
|
||||
argparser.add_argument('--custom_processor',
|
||||
type=CustomProcessor,
|
||||
choices=list(CustomProcessor))
|
||||
argparser.add_argument('rewrapper_path')
|
||||
argparser.add_argument('--input_list_paths')
|
||||
argparser.add_argument('--output_list_paths')
|
||||
argparser.add_argument('--exec_root')
|
||||
parsed_args, extra_args = argparser.parse_known_args()
|
||||
|
||||
# This script expects to be calling rewrapper.
|
||||
args = [parsed_args.rewrapper_path]
|
||||
|
||||
# Get the output files list.
|
||||
output_files = set()
|
||||
with open(parsed_args.output_list_paths, 'r') as file:
|
||||
for line in file:
|
||||
# Output files are relative to exec_root.
|
||||
output_file = _normalize_path(
|
||||
os.path.join(parsed_args.exec_root, line.rstrip('\n')))
|
||||
output_files.add(output_file)
|
||||
|
||||
# Scan for and add explicit inputs for rewrapper if necessary.
|
||||
# These should be in a new input list paths file, as using --inputs can fail
|
||||
# if the list is extremely large.
|
||||
if parsed_args.custom_processor == CustomProcessor.mojom_parser:
|
||||
root, ext = os.path.splitext(parsed_args.input_list_paths)
|
||||
extra_inputs = _get_mojom_parser_inputs(parsed_args.exec_root, output_files,
|
||||
extra_args)
|
||||
extra_input_list_path = '%s__extra%s' % (root, ext)
|
||||
with open(extra_input_list_path, 'w') as file:
|
||||
with open(parsed_args.input_list_paths, 'r') as inputs:
|
||||
file.write(inputs.read())
|
||||
file.write("\n".join(extra_inputs))
|
||||
args += ["--input_list_paths=%s" % extra_input_list_path]
|
||||
else:
|
||||
args += ["--input_list_paths=%s" % parsed_args.input_list_paths]
|
||||
|
||||
# Filter out --custom_processor= which is a flag for this script,
|
||||
# and filter out --input_list_paths= because we replace it above.
|
||||
# Pass on the rest of the args to rewrapper.
|
||||
args_rest = filter(lambda arg: '--custom_processor=' not in arg, sys.argv[2:])
|
||||
args += filter(lambda arg: '--input_list_paths=' not in arg, args_rest)
|
||||
|
||||
# Run rewrapper.
|
||||
proc = subprocess.run(args)
|
||||
return proc.returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -795,12 +795,6 @@ template("mojom") {
|
||||
}
|
||||
|
||||
action("${parser_target_name}_action") {
|
||||
allow_remote = true
|
||||
|
||||
# Use large worker to avoid OOM (b/325844324) and make it faster with
|
||||
# multiple cores.
|
||||
remote_worker = "large"
|
||||
custom_processor = "mojom_parser"
|
||||
script = mojom_parser_script
|
||||
inputs = mojom_parser_sources + ply_sources + [ build_metadata_filename ]
|
||||
sources = sources_list
|
||||
@ -943,7 +937,6 @@ template("mojom") {
|
||||
}
|
||||
|
||||
action(generator_cpp_message_ids_target_name) {
|
||||
allow_remote = true
|
||||
script = mojom_generator_script
|
||||
inputs = mojom_generator_sources + jinja2_sources
|
||||
sources = sources_list +
|
||||
@ -991,7 +984,6 @@ template("mojom") {
|
||||
generator_shared_target_name = "${target_name}_shared__generator"
|
||||
|
||||
action(generator_shared_target_name) {
|
||||
allow_remote = true
|
||||
visibility = [ ":*" ]
|
||||
script = mojom_generator_script
|
||||
inputs = mojom_generator_sources + jinja2_sources
|
||||
@ -1124,7 +1116,6 @@ template("mojom") {
|
||||
"${target_name}_mojolpm_proto_generator"
|
||||
|
||||
action(generator_mojolpm_proto_target_name) {
|
||||
allow_remote = true
|
||||
script = mojom_generator_script
|
||||
inputs = mojom_generator_sources + jinja2_sources
|
||||
sources =
|
||||
@ -1398,7 +1389,6 @@ template("mojom") {
|
||||
|
||||
# TODO(crbug.com/40758345): Investigate nondeterminism in Py3 builds.
|
||||
action(generator_target_name) {
|
||||
allow_remote = true
|
||||
visibility = [ ":*" ]
|
||||
script = mojom_generator_script
|
||||
inputs = mojom_generator_sources + jinja2_sources
|
||||
@ -1541,7 +1531,6 @@ template("mojom") {
|
||||
write_file(_typemap_config_filename, _rebased_typemap_configs, "json")
|
||||
|
||||
action(_typemap_validator_target_name) {
|
||||
allow_remote = true
|
||||
script = "$mojom_generator_root/validate_typemap_config.py"
|
||||
inputs = [ _typemap_config_filename ]
|
||||
outputs = [ _typemap_stamp_filename ]
|
||||
@ -1560,7 +1549,6 @@ template("mojom") {
|
||||
write_file(_ts_typemap_config_filename, ts_typemap_configs, "json")
|
||||
|
||||
action(_ts_typemap_validator_target_name) {
|
||||
allow_remote = true
|
||||
script = "$mojom_generator_root/validate_ts_typemap_config.py"
|
||||
inputs = [ _ts_typemap_config_filename ]
|
||||
outputs = [ _ts_typemap_stamp_filename ]
|
||||
@ -1572,7 +1560,6 @@ template("mojom") {
|
||||
}
|
||||
|
||||
action(type_mappings_target_name) {
|
||||
allow_remote = true
|
||||
inputs =
|
||||
mojom_generator_sources + jinja2_sources + [ _typemap_stamp_filename ]
|
||||
outputs = [ type_mappings_path ]
|
||||
|
||||
Reference in New Issue
Block a user