Revert "Reland "Reland "simplify clobber.py with gn clean command"""
This reverts commit97d1bc9922. Reason for revert: Can't `gn gen` without first fetching .gni files See crbug.com/1406628#c51 Original change's description: > Reland "Reland "simplify clobber.py with gn clean command"" > > This reverts commitddcf7df155. > > Reason for revert: the previous land fails to find gn.bat on Windows. > This CL includes a fix for the Windows error. > > Test: 'git cl presubmit' on Windows > > Original change's description: > > Revert "Reland "simplify clobber.py with gn clean command"" > > > > This reverts commitae46eb62b1. > > > > Reason for revert: gclient hooks fail on Windows. > > https://logs.chromium.org/logs/chromium/buildbucket/cr-buildbucket/8791843871664997953/+/u/gclient_runhooks__with_patch_/stdout > > > > Original change's description: > > > Reland "simplify clobber.py with gn clean command" > > > > > > This reverts commitc0eeb7eac0. > > > > > > Reason for revert: Specifying `--root` option to the gn commands fixes the error in gclient hooks. > > > > > > Original change's description: > > > > Revert "simplify clobber.py with gn clean command" > > > > > > > > This reverts commitca9403a4e7. > > > > > > > > Reason for revert: Folks can't run gclient sync on ToT, crbug.com/1406628#c36 > > > > > > > > Original change's description: > > > > > simplify clobber.py with gn clean command > > > > > > > > > > The previous implementation of clobber.py doesn't work well with symlinked or mounted build dirs. > > > > > This CL refactors the script to use `gn clean` command to work on those build dirs. > > > > > > > > > > Bug: 1406628 > > > > > Fixed: 1407028 > > > > > Change-Id: I17c61b7e67075cb1ccef79692632564c667566ad > > > > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4164159 > > > > > Reviewed-by: Takuto Ikuta <tikuta@chromium.org> > > > > > Commit-Queue: Junji Watanabe <jwata@google.com> > > > > > Auto-Submit: Junji Watanabe <jwata@google.com> > > > > > Cr-Commit-Position: refs/heads/main@{#1092301} > > > > > > > > Bug: 1406628 > > > > Change-Id: I0e41d9bf6283e9afdc5623af3ca76f94fb0ae1f7 > > > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4164783 > > > > Owners-Override: Victor Vianna <victorvianna@google.com> > > > > Commit-Queue: Victor Vianna <victorvianna@google.com> > > > > Quick-Run: Victor Vianna <victorvianna@google.com> > > > > Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com> > > > > Auto-Submit: Victor Vianna <victorvianna@google.com> > > > > Cr-Commit-Position: refs/heads/main@{#1092438} > > > > > > Bug: 1406628 > > > Change-Id: I40e071f4a86c9ccb30bc3a585738a29172c15a5e > > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4170263 > > > Commit-Queue: Junji Watanabe <jwata@google.com> > > > Reviewed-by: Takuto Ikuta <tikuta@chromium.org> > > > Cr-Commit-Position: refs/heads/main@{#1092913} > > > > Bug: 1406628 > > Change-Id: I9c43c8da5383992c183b0910ab190ee91447d550 > > No-Presubmit: true > > No-Tree-Checks: true > > No-Try: true > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4170585 > > Reviewed-by: Takuto Ikuta <tikuta@chromium.org> > > Commit-Queue: Junji Watanabe <jwata@google.com> > > Cr-Commit-Position: refs/heads/main@{#1092936} > > Bug: 1406628 > Change-Id: I723e640a2c1cf5e5e4cb126faf3069f3573cb833 > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4171018 > Reviewed-by: Takuto Ikuta <tikuta@chromium.org> > Commit-Queue: Junji Watanabe <jwata@google.com> > Cr-Commit-Position: refs/heads/main@{#1092952} Bug: 1406628 Change-Id: Iba79e29940adbfa220eab9903b8250d2ec438555 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4175154 Commit-Queue: Justin DeWitt <dewittj@chromium.org> Commit-Queue: Ben Pastene <bpastene@chromium.org> Owners-Override: Travis Skare <skare@chromium.org> Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com> Owners-Override: Justin DeWitt <dewittj@chromium.org> Reviewed-by: Travis Skare <skare@chromium.org> Cr-Commit-Position: refs/heads/main@{#1093594}
This commit is contained in:
Ben Pastene
committed by
Chromium LUCI CQ
Chromium LUCI CQ
parent
8015b2f428
commit
e9d888e2bc
122
build/clobber.py
122
build/clobber.py
@ -7,47 +7,119 @@
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
_SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
def extract_gn_build_commands(build_ninja_file):
|
||||
"""Extracts from a build.ninja the commands to run GN.
|
||||
|
||||
The commands to run GN are the gn rule and build.ninja build step at the
|
||||
top of the build.ninja file. We want to keep these when deleting GN builds
|
||||
since we want to preserve the command-line flags to GN.
|
||||
|
||||
On error, returns the empty string."""
|
||||
result = ""
|
||||
with open(build_ninja_file, 'r') as f:
|
||||
# Reads until the first empty line after the "build build.ninja:" target.
|
||||
# We assume everything before it necessary as well (eg the
|
||||
# "ninja_required_version" line).
|
||||
found_build_dot_ninja_target = False
|
||||
for line in f.readlines():
|
||||
result += line
|
||||
if line.startswith('build build.ninja:'):
|
||||
found_build_dot_ninja_target = True
|
||||
if found_build_dot_ninja_target and line[0] == '\n':
|
||||
return result
|
||||
return '' # We got to EOF and didn't find what we were looking for.
|
||||
|
||||
|
||||
def _gn_cmd(cmd, build_dir):
|
||||
gn_exe = 'gn'
|
||||
if sys.platform == 'win32':
|
||||
gn_exe += '.bat'
|
||||
return [gn_exe, cmd, '--root=%s' % _SRC_DIR, '-C', build_dir]
|
||||
def delete_dir(build_dir):
|
||||
if os.path.islink(build_dir):
|
||||
return
|
||||
# For unknown reasons (anti-virus?) rmtree of Chromium build directories
|
||||
# often fails on Windows.
|
||||
if sys.platform.startswith('win'):
|
||||
subprocess.check_call(['rmdir', '/s', '/q', build_dir], shell=True)
|
||||
else:
|
||||
shutil.rmtree(build_dir)
|
||||
|
||||
|
||||
def _generate_build_ninja(build_dir):
|
||||
gn_gen_cmd = _gn_cmd('gen', build_dir)
|
||||
print('Running %s' % ' '.join(gn_gen_cmd))
|
||||
subprocess.run(gn_gen_cmd, check=True)
|
||||
def delete_build_dir(build_dir):
|
||||
# GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
|
||||
build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
|
||||
if not os.path.exists(build_ninja_d_file):
|
||||
delete_dir(build_dir)
|
||||
return
|
||||
|
||||
# GN builds aren't automatically regenerated when you sync. To avoid
|
||||
# messing with the GN workflow, erase everything but the args file, and
|
||||
# write a dummy build.ninja file that will automatically rerun GN the next
|
||||
# time Ninja is run.
|
||||
build_ninja_file = os.path.join(build_dir, 'build.ninja')
|
||||
build_commands = extract_gn_build_commands(build_ninja_file)
|
||||
|
||||
def _clean_build_dir(build_dir):
|
||||
print('Cleaning %s' % build_dir)
|
||||
|
||||
gn_clean_cmd = _gn_cmd('clean', build_dir)
|
||||
print('Running %s' % ' '.join(gn_clean_cmd))
|
||||
try:
|
||||
subprocess.run(gn_clean_cmd, check=True)
|
||||
except Exception:
|
||||
# gn clean may fail when build.ninja is corrupted or missing.
|
||||
# Regenerate build.ninja and retry gn clean again.
|
||||
_generate_build_ninja(build_dir)
|
||||
print('Running %s' % ' '.join(gn_clean_cmd))
|
||||
subprocess.run(gn_clean_cmd, check=True)
|
||||
gn_args_file = os.path.join(build_dir, 'args.gn')
|
||||
with open(gn_args_file, 'r') as f:
|
||||
args_contents = f.read()
|
||||
except IOError:
|
||||
args_contents = ''
|
||||
|
||||
exception_during_rm = None
|
||||
try:
|
||||
# delete_dir and os.mkdir() may fail, such as when chrome.exe is running,
|
||||
# and we still want to restore args.gn/build.ninja/build.ninja.d, so catch
|
||||
# the exception and rethrow it later.
|
||||
delete_dir(build_dir)
|
||||
os.mkdir(build_dir)
|
||||
except Exception as e:
|
||||
exception_during_rm = e
|
||||
|
||||
# Put back the args file (if any).
|
||||
if args_contents != '':
|
||||
with open(gn_args_file, 'w') as f:
|
||||
f.write(args_contents)
|
||||
|
||||
# Write the build.ninja file sufficiently to regenerate itself.
|
||||
with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
|
||||
if build_commands != '':
|
||||
f.write(build_commands)
|
||||
else:
|
||||
# Couldn't parse the build.ninja file, write a default thing.
|
||||
f.write('''ninja_required_version = 1.7.2
|
||||
|
||||
rule gn
|
||||
command = gn -q gen //out/%s/
|
||||
description = Regenerating ninja files
|
||||
|
||||
build build.ninja: gn
|
||||
generator = 1
|
||||
depfile = build.ninja.d
|
||||
''' % (os.path.split(build_dir)[1]))
|
||||
|
||||
# Write a .d file for the build which references a nonexistant file. This
|
||||
# will make Ninja always mark the build as dirty.
|
||||
with open(build_ninja_d_file, 'w') as f:
|
||||
f.write('build.ninja: nonexistant_file.gn\n')
|
||||
|
||||
if exception_during_rm:
|
||||
# Rethrow the exception we caught earlier.
|
||||
raise exception_during_rm
|
||||
|
||||
|
||||
def clobber(out_dir):
|
||||
"""Clobber contents of build directory."""
|
||||
"""Clobber contents of build directory.
|
||||
|
||||
Don't delete the directory itself: some checkouts have the build directory
|
||||
mounted."""
|
||||
for f in os.listdir(out_dir):
|
||||
path = os.path.join(out_dir, f)
|
||||
if os.path.isdir(path):
|
||||
_clean_build_dir(path)
|
||||
if os.path.isfile(path):
|
||||
os.unlink(path)
|
||||
elif os.path.isdir(path):
|
||||
delete_build_dir(path)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@ -7,33 +7,125 @@ import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import clobber
|
||||
|
||||
|
||||
class TestClean(unittest.TestCase):
|
||||
class TestExtractBuildCommand(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_, self.build_ninja_path = tempfile.mkstemp(text=True)
|
||||
|
||||
def tearDown(self):
|
||||
os.remove(self.build_ninja_path)
|
||||
|
||||
def test_normal_extraction(self):
|
||||
build_ninja_file_contents = textwrap.dedent("""
|
||||
ninja_required_version = 1.7.2
|
||||
|
||||
rule gn
|
||||
command = ../../buildtools/gn --root=../.. -q --regeneration gen .
|
||||
pool = console
|
||||
description = Regenerating ninja files
|
||||
|
||||
build build.ninja.stamp: gn
|
||||
generator = 1
|
||||
depfile = build.ninja.d
|
||||
|
||||
build build.ninja: phony build.ninja.stamp
|
||||
generator = 1
|
||||
|
||||
pool build_toolchain_action_pool
|
||||
depth = 72
|
||||
|
||||
pool build_toolchain_link_pool
|
||||
depth = 23
|
||||
|
||||
subninja toolchain.ninja
|
||||
subninja clang_newlib_x64/toolchain.ninja
|
||||
subninja glibc_x64/toolchain.ninja
|
||||
subninja irt_x64/toolchain.ninja
|
||||
subninja nacl_bootstrap_x64/toolchain.ninja
|
||||
subninja newlib_pnacl/toolchain.ninja
|
||||
|
||||
build blink_python_tests: phony obj/blink_python_tests.stamp
|
||||
build blink_tests: phony obj/blink_tests.stamp
|
||||
|
||||
default all
|
||||
""") # Based off of a standard linux build dir.
|
||||
with open(self.build_ninja_path, 'w') as f:
|
||||
f.write(build_ninja_file_contents)
|
||||
|
||||
expected_build_ninja_file_contents = textwrap.dedent("""
|
||||
ninja_required_version = 1.7.2
|
||||
|
||||
rule gn
|
||||
command = ../../buildtools/gn --root=../.. -q --regeneration gen .
|
||||
pool = console
|
||||
description = Regenerating ninja files
|
||||
|
||||
build build.ninja.stamp: gn
|
||||
generator = 1
|
||||
depfile = build.ninja.d
|
||||
|
||||
build build.ninja: phony build.ninja.stamp
|
||||
generator = 1
|
||||
|
||||
""")
|
||||
|
||||
self.assertEqual(clobber.extract_gn_build_commands(self.build_ninja_path),
|
||||
expected_build_ninja_file_contents)
|
||||
|
||||
def test_unexpected_format(self):
|
||||
# No "build build.ninja:" line should make it return an empty string.
|
||||
build_ninja_file_contents = textwrap.dedent("""
|
||||
ninja_required_version = 1.7.2
|
||||
|
||||
rule gn
|
||||
command = ../../buildtools/gn --root=../.. -q --regeneration gen .
|
||||
pool = console
|
||||
description = Regenerating ninja files
|
||||
|
||||
subninja toolchain.ninja
|
||||
|
||||
build blink_python_tests: phony obj/blink_python_tests.stamp
|
||||
build blink_tests: phony obj/blink_tests.stamp
|
||||
|
||||
""")
|
||||
with open(self.build_ninja_path, 'w') as f:
|
||||
f.write(build_ninja_file_contents)
|
||||
|
||||
self.assertEqual(clobber.extract_gn_build_commands(self.build_ninja_path),
|
||||
'')
|
||||
|
||||
|
||||
class TestDelete(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.build_dir = tempfile.mkdtemp()
|
||||
|
||||
pathlib.Path(os.path.join(self.build_dir, 'build.ninja')).touch()
|
||||
pathlib.Path(os.path.join(self.build_dir, 'build.ninja.d')).touch()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.build_dir)
|
||||
|
||||
def test_gn_clean_ok(self):
|
||||
pathlib.Path(os.path.join(self.build_dir, 'build.ninja')).touch()
|
||||
pathlib.Path(os.path.join(self.build_dir, 'build.ninja.d')).touch()
|
||||
|
||||
def test_delete_build_dir_full(self):
|
||||
# Create a dummy file in the build dir and ensure it gets removed.
|
||||
dummy_file = os.path.join(self.build_dir, 'dummy')
|
||||
pathlib.Path(dummy_file).touch()
|
||||
|
||||
clobber._clean_build_dir(self.build_dir)
|
||||
clobber.delete_build_dir(self.build_dir)
|
||||
|
||||
self.assertFalse(os.path.exists(dummy_file))
|
||||
|
||||
def test_gn_clean_fail(self):
|
||||
# gn clean fails without build.ninja.
|
||||
# clean_build_dir() regenerates build.ninja internally.
|
||||
clobber._clean_build_dir(self.build_dir)
|
||||
def test_delete_build_dir_fail(self):
|
||||
# Make delete_dir() throw to ensure it's handled gracefully.
|
||||
|
||||
with mock.patch('clobber.delete_dir', side_effect=OSError):
|
||||
with self.assertRaises(OSError):
|
||||
clobber.delete_build_dir(self.build_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user