0

WebUI: Replace vulcanize with polymer-bundler.

Summary of changes needed for the migration:
 - Excluded URLs behavior is different than Vulcanize, need to explicitly list
   all URLs to be excluded (unlike Vulcanize, transitive deps of an excluded
   URL are not implicitly excluded).
 - Excluded URLs need to match exactly, unlike Vulcanize where foo.html would
   match chrome://some-path/foo.html.
 - Input HTML files need to have
   <html><head>...</head><body>...</body></html> tags.
 - A single invocation of poylmer-bundler produces both "basic" and
   "lazy" modules (achieved by using the --shell flag), unlike Vulcanize
   which required two separate invocations.
 - Using new --manifest-out instead of previous --out-request-list flag to
   generate depfiles for Ninja.
 - Using --rewrite-urls-in-templates to preserve previous rewriting behavior.
 - polymer-bundler does not blow up when it can't find a file (unlike Vulcanize,
   so check output manifest for any reported "missing" dependencies).
 - Removed some Windows specific hacks that don't seem necessary anymore.

BUG=731881
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:closure_compilation

Review-Url: https://codereview.chromium.org/2936333002
Cr-Commit-Position: refs/heads/master@{#488821}
This commit is contained in:
dpapad
2017-07-21 17:22:25 -07:00
committed by Commit Bot
parent af3918157d
commit 1e6a87f5d3
19 changed files with 199 additions and 184 deletions

@ -4,11 +4,11 @@ vulcanized_unbuilt = "vulcanized.unbuilt.html"
vulcanize("vulcanize") {
host = "bookmarks"
html_in_file = "bookmarks.html"
html_out_file = vulcanized_unbuilt
html_in_files = [ "bookmarks.html" ]
html_out_files = [ vulcanized_unbuilt ]
input = rebase_path(".", root_build_dir)
js_out_file = "crisper.js"
js_out_files = [ "crisper.js" ]
deps = []
}

@ -18,7 +18,7 @@
<body>
<bookmarks-app></bookmarks-app>
<link rel="import" href="chrome://resources/html/load_time_data.html">
<script src="chrome://resources/js/load_time_data.js"></script>
<script src="chrome://bookmarks/strings.js"></script>
<link rel="import" href="chrome://bookmarks/app.html">
</body>

@ -5,11 +5,11 @@ vulcanized_unbuilt = "vulcanized.unbuilt.html"
vulcanize("vulcanize") {
deps = []
host = "downloads"
html_in_file = "downloads.html"
html_out_file = vulcanized_unbuilt
html_in_files = [ "downloads.html" ]
html_out_files = [ vulcanized_unbuilt ]
input = rebase_path(".", root_build_dir)
insert_in_head = "<base href=chrome://downloads>"
js_out_file = "crisper.js"
js_out_files = [ "crisper.js" ]
}
polymer_css_build("build") {

@ -5,31 +5,24 @@ lazy_load_unbuilt = "lazy_load.vulcanized.unbuilt.html"
vulcanize("vulcanize_app") {
host = "history"
html_in_file = "app.html"
html_out_file = app_unbuilt
input = rebase_path(".", root_build_dir)
js_out_file = "app.crisper.js"
excludes = [
"chrome://resources/html/util.html",
"chrome://history/constants.html",
html_in_files = [
"app.html",
"lazy_load.html",
]
html_out_files = [
app_unbuilt,
lazy_load_unbuilt,
]
deps = []
}
vulcanize("vulcanize_lazy_load") {
host = "history"
html_in_file = "lazy_load.html"
html_out_file = lazy_load_unbuilt
input = rebase_path(".", root_build_dir)
js_out_file = "lazy_load.crisper.js"
js_out_files = [
"app.crisper.js",
"lazy_load.crisper.js",
]
excludes = [
"chrome://resources/html/util.html",
"chrome://history/constants.html",
"chrome://history/app.html",
]
deps = []
@ -46,6 +39,5 @@ polymer_css_build("build") {
]
deps = [
":vulcanize_app",
":vulcanize_lazy_load",
]
}

@ -1,7 +1,12 @@
<link rel="import" href="chrome://history/synced_device_manager.html">
<link rel="import" href="chrome://resources/cr_elements/cr_action_menu/cr_action_menu.html">
<link rel="import" href="chrome://resources/cr_elements/cr_dialog/cr_dialog.html">
<link rel="import" href="chrome://resources/cr_elements/cr_drawer/cr_drawer.html">
<link rel="import" href="chrome://resources/cr_elements/cr_toolbar/cr_toolbar_selection_overlay.html">
<link rel="import" href="chrome://resources/polymer/v1_0/paper-button/paper-button.html">
<link rel="import" href="chrome://resources/polymer/v1_0/paper-icon-button/paper-icon-button-light.html">
<html>
<head></head>
<body>
<link rel="import" href="chrome://history/synced_device_manager.html">
<link rel="import" href="chrome://resources/cr_elements/cr_action_menu/cr_action_menu.html">
<link rel="import" href="chrome://resources/cr_elements/cr_dialog/cr_dialog.html">
<link rel="import" href="chrome://resources/cr_elements/cr_drawer/cr_drawer.html">
<link rel="import" href="chrome://resources/cr_elements/cr_toolbar/cr_toolbar_selection_overlay.html">
<link rel="import" href="chrome://resources/polymer/v1_0/paper-button/paper-button.html">
<link rel="import" href="chrome://resources/polymer/v1_0/paper-icon-button/paper-icon-button-light.html">
</body>
</html>

@ -9,26 +9,20 @@ unpak_folder = "settings_resources.unpak"
vulcanize("vulcanize_app") {
host = "md-settings"
html_in_file = "settings.html"
html_out_file = app_unbuilt
insert_in_head = "<base href=\"chrome://\$i18n{hostname}\">"
input = rebase_path("$target_gen_dir/$unpak_folder", root_build_dir)
js_out_file = "crisper.js"
deps = [
":unpak",
html_in_files = [
"settings.html",
"lazy_load.html",
]
html_out_files = [
app_unbuilt,
lazy_load_unbuilt,
]
}
vulcanize("vulcanize_lazy_load") {
host = "md-settings"
html_in_file = "lazy_load.html"
html_out_file = lazy_load_unbuilt
insert_in_head = "<base href=\"chrome://\$i18n{hostname}\">"
input = rebase_path("$target_gen_dir/$unpak_folder", root_build_dir)
js_out_file = "lazy_load.crisper.js"
excludes = [ "/settings.html" ]
js_out_files = [
"crisper.js",
"lazy_load.crisper.js",
]
deps = [
":unpak",
@ -55,7 +49,6 @@ polymer_css_build("build") {
]
deps = [
":vulcanize_app",
":vulcanize_lazy_load",
]
}

@ -1,2 +1,2 @@
<script src="chrome://resources/js/load_time_data.js"></script>
<script src="/strings.js"></script>
<script src="strings.js"></script>

@ -1,15 +1,20 @@
<link rel="import" href="a11y_page/a11y_page.html">
<link rel="import" href="downloads_page/downloads_page.html">
<link rel="import" href="languages_page/languages_page.html">
<link rel="import" href="passwords_and_forms_page/passwords_and_forms_page.html">
<link rel="import" href="printing_page/printing_page.html">
<link rel="import" href="privacy_page/privacy_page.html">
<link rel="import" href="reset_page/reset_page.html">
<html>
<head></head>
<body>
<link rel="import" href="a11y_page/a11y_page.html">
<link rel="import" href="downloads_page/downloads_page.html">
<link rel="import" href="languages_page/languages_page.html">
<link rel="import" href="passwords_and_forms_page/passwords_and_forms_page.html">
<link rel="import" href="printing_page/printing_page.html">
<link rel="import" href="privacy_page/privacy_page.html">
<link rel="import" href="reset_page/reset_page.html">
<if expr="chromeos">
<link rel="import" href="date_time_page/date_time_page.html">
</if>
<if expr="chromeos">
<link rel="import" href="date_time_page/date_time_page.html">
</if>
<if expr="not chromeos">
<link rel="import" href="system_page/system_page.html">
</if>
<if expr="not chromeos">
<link rel="import" href="system_page/system_page.html">
</if>
</body>
</html>

@ -34,12 +34,16 @@ template("vulcanize") {
]
# This depfile is generated by vulcanize_gn.py
depfile = "${target_gen_dir}/${invoker.html_out_file}.d"
depfile = "${target_gen_dir}/${target_name}.d"
outputs = []
foreach(_out, invoker.html_out_files) {
outputs += [ "$target_gen_dir/$_out" ]
}
foreach(_out, invoker.js_out_files) {
outputs += [ "$target_gen_dir/$_out" ]
}
outputs = [
"$target_gen_dir/${invoker.html_out_file}",
"$target_gen_dir/${invoker.js_out_file}",
]
deps = invoker.deps
# Note that we have to manually pass the sources to our script if the
@ -47,12 +51,6 @@ template("vulcanize") {
args = [
"--host",
invoker.host,
"--html_in_file",
invoker.html_in_file,
"--html_out_file",
invoker.html_out_file,
"--js_out_file",
invoker.js_out_file,
"--input",
invoker.input,
"--out_folder",
@ -61,6 +59,10 @@ template("vulcanize") {
rebase_path(depfile, root_build_dir),
]
args += [ "--html_in_files" ] + invoker.html_in_files
args += [ "--html_out_files" ] + invoker.html_out_files
args += [ "--js_out_files" ] + invoker.js_out_files
if (defined(invoker.excludes)) {
args += [ "--exclude" ] + invoker.excludes
}

@ -5,9 +5,11 @@
import argparse
import itertools
import json
import os
import platform
import re
import shutil
import sys
import tempfile
@ -43,16 +45,20 @@ _POLYMER_PATH = os.path.join(
_VULCANIZE_BASE_ARGS = [
# These files are already combined and minified.
'--exclude', 'chrome://resources/html/polymer.html',
'--exclude', 'web-animations-next-lite.min.js',
'--exclude', 'chrome://resources/polymer/v1_0/polymer/polymer.html',
'--exclude', 'chrome://resources/polymer/v1_0/polymer/polymer-micro.html',
'--exclude', 'chrome://resources/polymer/v1_0/polymer/polymer-mini.html',
'--exclude', 'chrome://resources/polymer/v1_0/web-animations-js/' +
'web-animations-next-lite.min.js',
# These files are dynamically created by C++.
'--exclude', 'load_time_data.js',
'--exclude', 'strings.js',
'--exclude', 'text_defaults.css',
'--exclude', 'text_defaults_md.css',
'--exclude', 'chrome://resources/css/roboto.css',
'--exclude', 'chrome://resources/css/text_defaults.css',
'--exclude', 'chrome://resources/css/text_defaults_md.css',
'--exclude', 'chrome://resources/js/load_time_data.js',
'--inline-css',
'--inline-scripts',
'--rewrite-urls-in-templates',
'--strip-comments',
]
@ -77,32 +83,24 @@ def _undo_mapping(mappings, url):
# TODO(dbeam): can we make this stricter?
return url
def _request_list_path(out_path, html_out_file):
return os.path.join(out_path, html_out_file + '_requestlist.txt')
def _request_list_path(out_path, host):
return os.path.join(out_path, host + '_requestlist.txt')
# Get a list of all files that were bundled with Vulcanize and update the
# depfile accordingly such that Ninja knows when to trigger re-vulcanization.
def _update_dep_file(in_folder, args):
# Get a list of all files that were bundled with polymer-bundler and update the
# depfile accordingly such that Ninja knows when to re-trigger.
def _update_dep_file(in_folder, args, manifest):
in_path = os.path.join(_CWD, in_folder)
out_path = os.path.join(_CWD, args.out_folder)
# Prior call to vulcanize already generated the deps list, grab it from there.
request_list_path = _request_list_path(out_path, args.html_out_file)
request_list = open(request_list_path, 'r').read().splitlines()
# Gather the dependencies of all bundled root HTML files.
request_list = []
for html_file in manifest:
request_list += manifest[html_file]
if platform.system() == 'Windows':
# TODO(dbeam): UGH. For some reason Vulcanize is interpreting the target
# file path as a URL and using the drive letter (e.g. D:\) as a protocol.
# This is a little insane, but we're fixing here by normalizing case (which
# really shouldn't matter, these are all file paths and generally are all
# lower case) and writing from / to \ (file path) and then back again. This
# is compounded by NodeJS having a bug in url.resolve() that handles
# chrome:// protocol URLs poorly as well as us using startswith() to strip
# file paths (which isn't crazy awesome either). Don't remove unless you
# really really know what you're doing.
norm = lambda u: u.lower().replace('/', '\\')
request_list = [norm(u).replace(norm(in_path), '').replace('\\', '/')
for u in request_list]
# Add a slash in front of every dependency that is not a chrome:// URL, so
# that we can map it to the correct source file path below.
request_list = map(
lambda dep: '/' + dep if not dep.startswith('chrome://') else dep,
request_list)
# Undo the URL mappings applied by vulcanize to get file paths relative to
# current working directory.
@ -121,81 +119,86 @@ def _update_dep_file(in_folder, args):
deps = [d for d in deps if not d.startswith(filter_url)]
with open(os.path.join(_CWD, args.depfile), 'w') as f:
deps_file_header = os.path.join(args.out_folder, args.html_out_file)
deps_file_header = os.path.join(args.out_folder, args.html_out_files[0])
f.write(deps_file_header + ': ' + ' '.join(deps))
def _vulcanize(in_folder, args):
in_path = os.path.normpath(os.path.join(_CWD, in_folder))
out_path = os.path.join(_CWD, args.out_folder)
html_out_path = os.path.join(out_path, args.html_out_file)
js_out_path = os.path.join(out_path, args.js_out_file)
manifest_out_path = _request_list_path(out_path, args.host)
exclude_args = []
for f in args.exclude or []:
exclude_args.append('--exclude')
exclude_args.append(f)
output = node.RunNode(
[node_modules.PathToVulcanize()] +
in_html_args = []
for f in args.html_in_files:
in_html_args.append('--in-html')
in_html_args.append(f)
tmp_out_dir = os.path.join(out_path, 'bundled')
node.RunNode(
[node_modules.PathToBundler()] +
_VULCANIZE_BASE_ARGS + _VULCANIZE_REDIRECT_ARGS + exclude_args +
['--out-request-list', _request_list_path(out_path, args.html_out_file),
'--redirect', '"/|%s"' % in_path,
[# This file is dynamically created by C++. Need to specify an exclusion
# URL for both the relative URL and chrome:// URL syntax.
'--exclude', 'strings.js',
'--exclude', 'chrome://%s/strings.js' % args.host,
'--manifest-out', manifest_out_path,
'--root', in_path,
'--redirect', '"chrome://%s/|%s"' % (args.host, in_path),
# TODO(dpapad): Figure out why vulcanize treats the input path
# differently on Windows VS Linux/Mac.
os.path.join(
in_path if platform.system() == 'Windows' else os.sep,
args.html_in_file)])
'--out-dir', os.path.relpath(tmp_out_dir, _CWD),
'--shell', args.html_in_files[0],
] + in_html_args)
# Grit includes are not supported, use HTML imports instead.
output = output.replace('<include src="', '<include src-disabled="')
for index, html_file in enumerate(args.html_in_files):
with open(
os.path.join(os.path.relpath(tmp_out_dir, _CWD), html_file), 'r') as f:
output = f.read()
if args.insert_in_head:
assert '<head>' in output
# NOTE(dbeam): Vulcanize eats <base> tags after processing. This undoes
# that by adding a <base> tag to the (post-processed) generated output.
output = output.replace('<head>', '<head>' + args.insert_in_head)
# Grit includes are not supported, use HTML imports instead.
output = output.replace('<include src="', '<include src-disabled="')
crisper_input = tempfile.NamedTemporaryFile(mode='wt+', delete=False)
crisper_input.write(output)
crisper_input.close()
if args.insert_in_head:
assert '<head>' in output
# NOTE(dbeam): polymer-bundler eats <base> tags after processing. This
# undoes that by adding a <base> tag to the (post-processed) generated
# output.
output = output.replace('<head>', '<head>' + args.insert_in_head)
crisper_output = tempfile.NamedTemporaryFile(mode='wt+', delete=False)
crisper_output.close()
# Open file again with 'w' such that the previous contents are overwritten.
with open(
os.path.join(os.path.relpath(tmp_out_dir, _CWD), html_file), 'w') as f:
f.write(output)
f.close()
try:
node.RunNode([node_modules.PathToCrisper(),
'--source', crisper_input.name,
'--script-in-head', 'false',
'--only-split',
'--html', html_out_path,
'--js', crisper_output.name])
for index, html_in_file in enumerate(args.html_in_files):
html_out_file = args.html_out_files[index]
js_out_file = args.js_out_files[index]
# Crisper by default inserts a <script> tag with the name of the --js file,
# but since we are using a temporary file, need to manually insert a
# <script> tag with the correct final filename (in combination with
# --only-split flag). There is no way currently to manually specify the
# <script> tag's path, see https://github.com/PolymerLabs/crisper/issues/46.
with open(html_out_path, 'r+') as f:
data = f.read()
new_data = data.replace(
'</body></html>',
'<script src="' + args.js_out_file + '"></script></body></html>')
assert new_data != data, 'Expected to find </body></html> token.'
f.seek(0)
f.write(new_data)
f.truncate()
# Run crisper to separate the JS from the HTML file.
node.RunNode([node_modules.PathToCrisper(),
'--source', os.path.join(tmp_out_dir, html_in_file),
'--script-in-head', 'false',
'--html', os.path.join(tmp_out_dir, html_out_file),
'--js', os.path.join(tmp_out_dir, js_out_file)])
node.RunNode([node_modules.PathToUglify(), crisper_output.name,
'--comments', '"/Copyright|license|LICENSE|\<\/?if/"',
'--output', js_out_path])
# Move the HTML file to its final destination.
shutil.copy(os.path.join(tmp_out_dir, html_out_file), out_path)
# Pass the JS file through Uglify and write the output to its final
# destination.
node.RunNode([node_modules.PathToUglify(),
os.path.join(tmp_out_dir, js_out_file),
'--comments', '"/Copyright|license|LICENSE|\<\/?if/"',
'--output', os.path.join(out_path, js_out_file)])
finally:
if os.path.exists(crisper_input.name):
os.remove(crisper_input.name)
if os.path.exists(crisper_output.name):
os.remove(crisper_output.name)
shutil.rmtree(tmp_out_dir)
return manifest_out_path
def main(argv):
@ -203,11 +206,11 @@ def main(argv):
parser.add_argument('--depfile', required=True)
parser.add_argument('--exclude', nargs='*')
parser.add_argument('--host', required=True)
parser.add_argument('--html_in_file', required=True)
parser.add_argument('--html_out_file', required=True)
parser.add_argument('--html_in_files', nargs='*', required=True)
parser.add_argument('--html_out_files', nargs='*', required=True)
parser.add_argument('--input', required=True)
parser.add_argument('--insert_in_head')
parser.add_argument('--js_out_file', required=True)
parser.add_argument('--js_out_files', nargs='*', required=True)
parser.add_argument('--out_folder', required=True)
args = parser.parse_args(argv)
@ -218,8 +221,20 @@ def main(argv):
args.input = os.path.normpath(args.input)
args.out_folder = os.path.normpath(args.out_folder)
_vulcanize(args.input, args)
_update_dep_file(args.input, args)
manifest_out_path = _vulcanize(args.input, args)
# Prior call to _vulcanize() generated an output manifest file, containing
# information about all files that were bundled. Grab it from there.
manifest = json.loads(open(manifest_out_path, 'r').read())
# polymer-bundler reports any missing files in the output manifest, instead of
# directly failing. Ensure that no such files were encountered.
if '_missing' in manifest:
raise Exception(
'polymer-bundler could not find files for the following URLs:\n' +
'\n'.join(manifest['_missing']))
_update_dep_file(args.input, args, manifest)
if __name__ == '__main__':

@ -158,7 +158,8 @@ PolymerTest.loadScript = function(src) {
*/
PolymerTest.clearBody = function() {
// Save the div where vulcanize inlines content before clearing the page.
var vulcanizeDiv = document.querySelector('body > div[hidden][by-vulcanize]');
var vulcanizeDiv = document.querySelector(
'body > div[hidden][by-polymer-bundler]');
document.body.innerHTML = '';
if (vulcanizeDiv)
document.body.appendChild(vulcanizeDiv);

@ -27,7 +27,7 @@ continue (i.e. script execution may be paused).
```
To reduce this latency, Chrome uses a tool created by the Polymer project named
[vulcanize](https://github.com/Polymer/polymer-bundler/tree/1.x). It processes
[polymer-bundler](https://github.com/Polymer/polymer-bundler). It processes
a page starting from a URL entry point and inlines resources the first time
they're encountered. This greatly decreases latency due to HTML imports.

@ -14,4 +14,4 @@ Use update_node_binaries to update Node binaries and update_npm_deps to
update NPM dependencies.
Local Modifications:
See patch_vulcanize.diff
See chromium_polymer_bundler.patch

@ -0,0 +1,13 @@
diff --git a/lib/constants.js b/lib/constants.js
index 6e1a557..23d7f7c 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -18,7 +18,7 @@ exports.default = {
ABS_URL: /(^\/)|(^#)|(^[\w-\d]*:)/,
URL: /url\([^)]*\)/g,
URL_ATTR: ['href', 'src', 'action', 'style', 'assetpath'],
- URL_TEMPLATE: '{{.*}}|\\[\\[.*\\]\\]',
+ URL_TEMPLATE: '{{.*}}|\\[\\[.*\\]\\]|\\$i18n[^{]*{[^}]*}',
OLD_POLYMER: 'This version of polymer-bundler is not compatible with Polymer < 0.8. Please use vulcanize 0.7.x.'
};
//# sourceMappingURL=constants.js.map

@ -1,12 +0,0 @@
diff --git a/lib/constants.js b/lib/constants.js
index 21e1380..b6a353a 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -13,6 +13,6 @@ module.exports = {
ABS_URL: /(^\/)|(^#)|(^[\w-\d]*:)/,
URL: /url\([^)]*\)/g,
URL_ATTR: ['href', 'src', 'action', 'style', 'assetpath'],
- URL_TEMPLATE: '{{.*}}|\\[\\[.*\\]\\]',
+ URL_TEMPLATE: '{{.*}}|\\[\\[.*\\]\\]|\\$i18n[^{]*{[^}]*}',
OLD_POLYMER: 'This version of vulcanize is not compatible with Polymer < 0.8. Please use vulcanize 0.7.x.'
};

@ -13,8 +13,9 @@ def _path_in_node_modules(*args):
return os_path.join(NODE_MODULES, *args)
def PathToVulcanize():
return _path_in_node_modules('vulcanize', 'bin', 'vulcanize')
def PathToBundler():
return _path_in_node_modules(
'polymer-bundler', 'lib', 'bin', 'polymer-bundler')
def PathToCrisper():

@ -1 +1 @@
6dbd64a4fc98b0f3b72625c1585d95f9e9b6d71a
050c85d20f7cedd7f5c39533c1ba89dcdfa56a08

@ -5,8 +5,8 @@
"dependencies": {
"crisper": "2.0.2",
"eslint": "3.19.0",
"polymer-bundler": "3.0.1",
"polymer-css-build": "0.1.2",
"uglify-es": "3.0.15",
"vulcanize": "1.15.4"
"uglify-es": "3.0.15"
}
}

@ -18,8 +18,8 @@ rm -rf node_modules
npm install --no-bin-links --only=prod
# Apply local patch to vulcanize.
patch -d node_modules/vulcanize/ -p1 < chromium_vulcanize.patch
# Apply local patch to polymer-bundler.
patch -d node_modules/polymer-bundler/ -p1 < chromium_polymer_bundler.patch
rsync -c --delete -r -q --exclude-from="npm_exclude.txt" \
--prune-empty-dirs "node_modules/" "node_modules_filtered/"