buildtools: rename fetch_reclient_cfgs.py to configure_reclient_cfgs.py

To generate reproxy.cfg for non-Googler, but use rewrapper configs in
https://source.chromium.org/chromium/chromium/src/+/main:buildtools/reclient_cfgs/linux/
use case, let me change the name of the script.

I'll change the script not to download reclient cfgs CIPD package in
some cases depending on given gclient variables for external customers.

Bug: b/291868532
Change-Id: I7c88bf40e9f5349c8609fcc45870753777d6ed75
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4777755
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
Reviewed-by: Junji Watanabe <jwata@google.com>
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1184508}

DEPS

@@ -5676,13 +5676,13 @@ hooks = [
                '--bucket', 'chromium-style-perftest',
                '-d', 'src/third_party/blink/renderer/core/css/perftest_data'],
   },
-  # Download remote exec cfg files
+  # Configure remote exec cfg files
   {
-    'name': 'fetch_reclient_cfgs',
+    'name': 'configure_reclient_cfgs',
     'pattern': '.',
     'condition': 'download_remoteexec_cfg',
     'action': ['python3',
-               'src/buildtools/reclient_cfgs/fetch_reclient_cfgs.py',
+               'src/buildtools/reclient_cfgs/configure_reclient_cfgs.py',
                '--rbe_instance',
                Var('rbe_instance'),
                '--reproxy_cfg_template',

buildtools/reclient_cfgs/configure_reclient_cfgs.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# Copyright 2021 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script is used to fetch reclient cfgs."""
+
+import argparse
+import glob
+import logging
+import os
+import posixpath
+import re
+import shutil
+import string
+import subprocess
+import sys
+
+THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+CHROMIUM_SRC = os.path.abspath(os.path.join(THIS_DIR,'..','..'))
+
+REPROXY_CFG_HEADER = """# AUTOGENERATED FILE - DO NOT EDIT
+# Generated by configure_reclient_cfgs.py
+# To edit:
+# Update reproxy_cfg_templates/$reproxy_cfg_template
+# And run 'gclient sync' or 'gclient runhooks'
+"""
+
+def ClangRevision():
+    sys.path.insert(0, os.path.join(CHROMIUM_SRC,
+                                    'tools', 'clang', 'scripts'))
+    import update
+    sys.path.pop(0)
+    return update.PACKAGE_VERSION
+
+def NaclRevision():
+    nacl_dir = os.path.join(CHROMIUM_SRC, 'native_client')
+    if not os.path.exists(nacl_dir):
+      return None
+    return subprocess.check_output(
+        ['git', 'log', '-1', '--format=%H'],
+        cwd= nacl_dir, shell=os.name == 'nt',
+    ).decode('utf-8').strip()
+
+class CipdError(Exception):
+  """Raised by configure_reclient_cfgs on fatal cipd error."""
+
+class CipdAuthError(CipdError):
+  """Raised by configure_reclient_cfgs on cipd auth error."""
+
+def CipdEnsure(pkg_name, ref, directory, quiet):
+    print('ensure %s %s in %s' % (pkg_name, ref, directory))
+    log_level = 'warning' if quiet else 'info'
+    ensure_file = """
+$ParanoidMode CheckIntegrity
+{pkg} {ref}
+""".format(pkg=pkg_name, ref=ref)
+    try:
+      output = subprocess.check_output(
+          ' '.join(['cipd', 'ensure', '-log-level=' + log_level,
+                    '-root', directory, '-ensure-file', '-']),
+          shell=True, input=ensure_file, stderr=subprocess.STDOUT,
+          universal_newlines=True)
+      logging.info(output)
+    except subprocess.CalledProcessError as e:
+      if not IsCipdLoggedIn():
+         raise CipdAuthError(e.output) from e
+      raise CipdError(e.output) from e
+
+def IsCipdLoggedIn():
+    ret = subprocess.call(
+       ['cipd', 'auth-info'],
+       stderr=subprocess.PIPE,
+       stdout=subprocess.DEVNULL)
+    return ret == 0
+
+def RbeProjectFromInstance(instance):
+    m = re.fullmatch(r'projects/([-\w]+)/instances/[-\w]+', instance)
+    if not m:
+        return None
+    return m.group(1)
+
+def GenerateReproxyCfg(reproxy_cfg_template, rbe_instance, rbe_project):
+    tmpl_path = os.path.join(
+        THIS_DIR, 'reproxy_cfg_templates', reproxy_cfg_template)
+    logging.info(f'generate reproxy.cfg using {tmpl_path}')
+    if not os.path.isfile(tmpl_path):
+        logging.warning(f"{tmpl_path} does not exist")
+        return False
+    with open(tmpl_path) as f:
+      reproxy_cfg_tmpl = string.Template(REPROXY_CFG_HEADER+f.read())
+    depsscanner_address = 'exec://' + os.path.join(CHROMIUM_SRC,
+                                                   'buildtools',
+                                                   'reclient',
+                                                   'scandeps_server')
+    if sys.platform.startswith('win'):
+       depsscanner_address += ".exe"
+    reproxy_cfg = reproxy_cfg_tmpl.substitute({
+      'rbe_instance': rbe_instance,
+      'rbe_project': rbe_project,
+      'reproxy_cfg_template': reproxy_cfg_template,
+      'depsscanner_address': depsscanner_address,
+    })
+    reproxy_cfg_path = os.path.join(THIS_DIR, 'reproxy.cfg')
+    with open(reproxy_cfg_path, 'w') as f:
+      f.write(reproxy_cfg)
+    return True
+
+
+def RequestCipdAuthentication():
+  """Requests that the user authenticate to access CIPD packages."""
+
+  print('Access to remoteexec config CIPD package requires authentication.')
+  print('-----------------------------------------------------------------')
+  print()
+  print('I\'m sorry for the hassle, but you may need to do a one-time manual')
+  print('authentication. Please run:')
+  print()
+  print('    cipd auth-login')
+  print()
+  print('and follow the instructions.')
+  print()
+  print('NOTE: Use your google.com credentials, not chromium.org.')
+  print()
+  print('-----------------------------------------------------------------')
+  print()
+  sys.stdout.flush()
+
+def main():
+    parser = argparse.ArgumentParser(
+       description='configure reclient cfgs',
+       formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument(
+        '--rewrapper_cfg_project', '--rbe_project',
+        help='RBE instance project id for rewrapper configs. '
+             'Only specify if different from --rbe_instance\n'
+             'TODO(b/270201776) --rbe_project is deprecated, '
+             'remove once no more usages exist')
+    parser.add_argument(
+        '--reproxy_cfg_template',
+        help='If set REPROXY_CFG_TEMPLATE will be used to generate '
+             'reproxy.cfg. --rbe_instance must be specified.')
+    parser.add_argument('--rbe_instance',
+                        help='RBE instance for rewrapper and reproxy configs',
+                        default=os.environ.get('RBE_instance'))
+    parser.add_argument('--cipd_prefix',
+                        help='cipd package name prefix',
+                        default='infra_internal/rbe/reclient_cfgs')
+    parser.add_argument(
+        '--quiet',
+        help='Suppresses info logs',
+        action='store_true')
+
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.WARNING if args.quiet else logging.INFO,
+                        format="%(message)s")
+
+    if not args.rewrapper_cfg_project and not args.rbe_instance:
+        logging.error(
+           'At least one of --rbe_instance and --rewrapper_cfg_project '
+           'must be provided')
+        return 1
+
+    rbe_project = args.rewrapper_cfg_project
+    if not args.rewrapper_cfg_project:
+        rbe_project = RbeProjectFromInstance(args.rbe_instance)
+
+    if args.reproxy_cfg_template:
+        if not args.rbe_instance:
+            logging.error(
+              '--rbe_instance is required if --reproxy_cfg_template is set')
+            return 1
+        if not GenerateReproxyCfg(
+          args.reproxy_cfg_template, args.rbe_instance, rbe_project):
+           return 1
+
+    logging.info('fetch reclient_cfgs for RBE project %s...' % rbe_project)
+
+    cipd_prefix = posixpath.join(args.cipd_prefix, rbe_project)
+
+    tool_revisions = {
+        'chromium-browser-clang': ClangRevision(),
+        'nacl': NaclRevision(),
+        'python': '3.8.0',
+    }
+    for toolchain in tool_revisions:
+      revision = tool_revisions[toolchain]
+      if not revision:
+        logging.info('failed to detect %s revision' % toolchain)
+        continue
+
+      toolchain_root = os.path.join(THIS_DIR, toolchain)
+      cipd_ref = 'revision/' + revision
+      # 'cipd ensure' initializes the directory.
+      try:
+        CipdEnsure(posixpath.join(cipd_prefix, toolchain),
+                    ref=cipd_ref,
+                    directory=toolchain_root,
+                    quiet=args.quiet)
+      except CipdAuthError as e:
+        RequestCipdAuthentication()
+        return 1
+      except CipdError as e:
+        logging.error(e)
+        return 1
+      # support legacy (win-cross-experiments) and new (win-cross)
+      # TODO(crbug.com/1407557): drop -experiments support
+      wcedir = os.path.join(THIS_DIR, 'win-cross', toolchain)
+      if not os.path.exists(wcedir):
+          os.makedirs(wcedir, mode=0o755)
+      for win_cross_cfg_dir in ['win-cross','win-cross-experiments']:
+          if os.path.exists(os.path.join(toolchain_root, win_cross_cfg_dir)):
+              # copy in win-cross*/toolchain
+              # as windows may not use symlinks.
+              for cfg in glob.glob(os.path.join(toolchain_root,
+                                                win_cross_cfg_dir,
+                                                '*.cfg')):
+                  fname = os.path.join(wcedir, os.path.basename(cfg))
+                  if os.path.exists(fname):
+                    os.chmod(fname, 0o777)
+                    os.remove(fname)
+                  logging.info('Copy from %s to %s...' % (cfg, fname))
+                  shutil.copy(cfg, fname)
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

buildtools/reclient_cfgs/fetch_reclient_cfgs.py

@@ -3,227 +3,15 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""This script is used to fetch reclient cfgs."""
+"""This script is used to fetch reclient cfgs.
 
-import argparse
-import glob
-import logging
-import os
-import posixpath
-import re
-import shutil
-import string
-import subprocess
-import sys
-
-THIS_DIR = os.path.abspath(os.path.dirname(__file__))
-CHROMIUM_SRC = os.path.abspath(os.path.join(THIS_DIR,'..','..'))
-
-REPROXY_CFG_HEADER = """# AUTOGENERATED FILE - DO NOT EDIT
-# Generated by fetch_reclient_cfgs.py
-# To edit:
-# Update reproxy_cfg_templates/$reproxy_cfg_template
-# And run 'gclient sync' or 'gclient runhooks'
+This is renamed to configure_reclient_cfgs.py, delete this file if there are no
+reference to this file.
 """
 
-def ClangRevision():
-    sys.path.insert(0, os.path.join(CHROMIUM_SRC,
-                                    'tools', 'clang', 'scripts'))
-    import update
-    sys.path.pop(0)
-    return update.PACKAGE_VERSION
+import sys
 
-def NaclRevision():
-    nacl_dir = os.path.join(CHROMIUM_SRC, 'native_client')
-    if not os.path.exists(nacl_dir):
-      return None
-    return subprocess.check_output(
-        ['git', 'log', '-1', '--format=%H'],
-        cwd= nacl_dir, shell=os.name == 'nt',
-    ).decode('utf-8').strip()
-
-class CipdError(Exception):
-  """Raised by fetch_reclient_cfgs on fatal cipd error."""
-
-class CipdAuthError(CipdError):
-  """Raised by fetch_reclient_cfgs on cipd auth error."""
-
-def CipdEnsure(pkg_name, ref, directory, quiet):
-    print('ensure %s %s in %s' % (pkg_name, ref, directory))
-    log_level = 'warning' if quiet else 'info'
-    ensure_file = """
-$ParanoidMode CheckIntegrity
-{pkg} {ref}
-""".format(pkg=pkg_name, ref=ref)
-    try:
-      output = subprocess.check_output(
-          ' '.join(['cipd', 'ensure', '-log-level=' + log_level,
-                    '-root', directory, '-ensure-file', '-']),
-          shell=True, input=ensure_file, stderr=subprocess.STDOUT,
-          universal_newlines=True)
-      logging.info(output)
-    except subprocess.CalledProcessError as e:
-      if not IsCipdLoggedIn():
-         raise CipdAuthError(e.output) from e
-      raise CipdError(e.output) from e
-
-def IsCipdLoggedIn():
-    ret = subprocess.call(
-       ['cipd', 'auth-info'],
-       stderr=subprocess.PIPE,
-       stdout=subprocess.DEVNULL)
-    return ret == 0
-
-def RbeProjectFromInstance(instance):
-    m = re.fullmatch(r'projects/([-\w]+)/instances/[-\w]+', instance)
-    if not m:
-        return None
-    return m.group(1)
-
-def GenerateReproxyCfg(reproxy_cfg_template, rbe_instance, rbe_project):
-    tmpl_path = os.path.join(
-        THIS_DIR, 'reproxy_cfg_templates', reproxy_cfg_template)
-    logging.info(f'generate reproxy.cfg using {tmpl_path}')
-    if not os.path.isfile(tmpl_path):
-        logging.warning(f"{tmpl_path} does not exist")
-        return False
-    with open(tmpl_path) as f:
-      reproxy_cfg_tmpl = string.Template(REPROXY_CFG_HEADER+f.read())
-    depsscanner_address = 'exec://' + os.path.join(CHROMIUM_SRC,
-                                                   'buildtools',
-                                                   'reclient',
-                                                   'scandeps_server')
-    if sys.platform.startswith('win'):
-       depsscanner_address += ".exe"
-    reproxy_cfg = reproxy_cfg_tmpl.substitute({
-      'rbe_instance': rbe_instance,
-      'rbe_project': rbe_project,
-      'reproxy_cfg_template': reproxy_cfg_template,
-      'depsscanner_address': depsscanner_address,
-    })
-    reproxy_cfg_path = os.path.join(THIS_DIR, 'reproxy.cfg')
-    with open(reproxy_cfg_path, 'w') as f:
-      f.write(reproxy_cfg)
-    return True
-
-
-def RequestCipdAuthentication():
-  """Requests that the user authenticate to access CIPD packages."""
-
-  print('Access to remoteexec config CIPD package requires authentication.')
-  print('-----------------------------------------------------------------')
-  print()
-  print('I\'m sorry for the hassle, but you may need to do a one-time manual')
-  print('authentication. Please run:')
-  print()
-  print('    cipd auth-login')
-  print()
-  print('and follow the instructions.')
-  print()
-  print('NOTE: Use your google.com credentials, not chromium.org.')
-  print()
-  print('-----------------------------------------------------------------')
-  print()
-  sys.stdout.flush()
-
-def main():
-    parser = argparse.ArgumentParser(
-       description='fetch reclient cfgs',
-       formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument(
-        '--rewrapper_cfg_project', '--rbe_project',
-        help='RBE instance project id for rewrapper configs. '
-             'Only specify if different from --rbe_instance\n'
-             'TODO(b/270201776) --rbe_project is deprecated, '
-             'remove once no more usages exist')
-    parser.add_argument(
-        '--reproxy_cfg_template',
-        help='If set REPROXY_CFG_TEMPLATE will be used to generate '
-             'reproxy.cfg. --rbe_instance must be specified.')
-    parser.add_argument('--rbe_instance',
-                        help='RBE instance for rewrapper and reproxy configs',
-                        default=os.environ.get('RBE_instance'))
-    parser.add_argument('--cipd_prefix',
-                        help='cipd package name prefix',
-                        default='infra_internal/rbe/reclient_cfgs')
-    parser.add_argument(
-        '--quiet',
-        help='Suppresses info logs',
-        action='store_true')
-
-    args = parser.parse_args()
-
-    logging.basicConfig(level=logging.WARNING if args.quiet else logging.INFO,
-                        format="%(message)s")
-
-    if not args.rewrapper_cfg_project and not args.rbe_instance:
-        logging.error(
-           'At least one of --rbe_instance and --rewrapper_cfg_project '
-           'must be provided')
-        return 1
-
-    rbe_project = args.rewrapper_cfg_project
-    if not args.rewrapper_cfg_project:
-        rbe_project = RbeProjectFromInstance(args.rbe_instance)
-
-    if args.reproxy_cfg_template:
-        if not args.rbe_instance:
-            logging.error(
-              '--rbe_instance is required if --reproxy_cfg_template is set')
-            return 1
-        if not GenerateReproxyCfg(
-          args.reproxy_cfg_template, args.rbe_instance, rbe_project):
-           return 1
-
-    logging.info('fetch reclient_cfgs for RBE project %s...' % rbe_project)
-
-    cipd_prefix = posixpath.join(args.cipd_prefix, rbe_project)
-
-    tool_revisions = {
-        'chromium-browser-clang': ClangRevision(),
-        'nacl': NaclRevision(),
-        'python': '3.8.0',
-    }
-    for toolchain in tool_revisions:
-      revision = tool_revisions[toolchain]
-      if not revision:
-        logging.info('failed to detect %s revision' % toolchain)
-        continue
-
-      toolchain_root = os.path.join(THIS_DIR, toolchain)
-      cipd_ref = 'revision/' + revision
-      # 'cipd ensure' initializes the directory.
-      try:
-        CipdEnsure(posixpath.join(cipd_prefix, toolchain),
-                    ref=cipd_ref,
-                    directory=toolchain_root,
-                    quiet=args.quiet)
-      except CipdAuthError as e:
-        RequestCipdAuthentication()
-        return 1
-      except CipdError as e:
-        logging.error(e)
-        return 1
-      # support legacy (win-cross-experiments) and new (win-cross)
-      # TODO(crbug.com/1407557): drop -experiments support
-      wcedir = os.path.join(THIS_DIR, 'win-cross', toolchain)
-      if not os.path.exists(wcedir):
-          os.makedirs(wcedir, mode=0o755)
-      for win_cross_cfg_dir in ['win-cross','win-cross-experiments']:
-          if os.path.exists(os.path.join(toolchain_root, win_cross_cfg_dir)):
-              # copy in win-cross*/toolchain
-              # as windows may not use symlinks.
-              for cfg in glob.glob(os.path.join(toolchain_root,
-                                                win_cross_cfg_dir,
-                                                '*.cfg')):
-                  fname = os.path.join(wcedir, os.path.basename(cfg))
-                  if os.path.exists(fname):
-                    os.chmod(fname, 0o777)
-                    os.remove(fname)
-                  logging.info('Copy from %s to %s...' % (cfg, fname))
-                  shutil.copy(cfg, fname)
-
-    return 0
+import configure_reclient_cfgs
 
 if __name__ == '__main__':
-    sys.exit(main())
+    sys.exit(configure_reclient_cfgs.main())