chromium/src
0
Fork 0
Code Activity

Reland "Migrate rust hook to deps and create sync_deps.py"

Browse Source 
This is a reland of commit 2af7751ec4, with the
following changes:
* update_rust - use VERSION instead of INSTALLED_VERSION

gclient ensures that the rust package is not partially extracted, so
INSTALLED_VERSION is no longer needed.


Original change's description:
> Migrate rust hook to deps and create sync_deps.py
>
> After someone runs upload_revision.py, triggers the packager bots,
> and manually copies the packages to prod, they will now run
> sync_deps.py to update the DEPS file. After doing that they can
> trigger a CQ run to test the new packages.
>
> Bug: b/338612245
> Change-Id: Ie95efbf2049e651192d52ca7531c20125b3dd537
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5536220
> Reviewed-by: Hans Wennborg <hans@chromium.org>
> Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#1304616}

Bug: b/338612245
Change-Id: I2038e57a2e767eea10df6e2b72b6d2e2344d630e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5560825
Reviewed-by: Hans Wennborg <hans@chromium.org>
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: danakj <danakj@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1305165}
This commit is contained in:
Josip Sokcevic
2024-05-23 17:02:19 +00:00
committed by Chromium LUCI CQ
parent 5c897e0ed5
commit 6f00b586dd
4 changed files with 118 additions and 13 deletions

42
DEPS

@ -540,6 +540,7 @@ allowed_hosts = [
# TODO(337061377): Move into a separate allowed gcs bucket list.
'chromium-ads-detection',
'chromium-browser-clang',
'chromium-clang-format',
'chromium-nodejs',
'chromium-tfhub-models',
@ -713,6 +714,41 @@ deps = {
},
],
},
# Update prebuilt Rust toolchain.
'src/third_party/rust-toolchain': {
'dep_type': 'gcs',
'bucket': 'chromium-browser-clang',
'objects': [
{
'object_name': 'Linux_x64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'e7e236fe6e61b7a41ba6b1e4bc64e91119359cdddce268597c6328b9fea8d76d',
'size_bytes': 151082700,
'generation': 1715715529295267,
'condition': 'host_os == "linux" and non_git_source',
},
{
'object_name': 'Mac/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '3860d592e7f6a853aa06a00642f4110132c7af7f2b91d7bdd399d00dbd5177f6',
'size_bytes': 143827372,
'generation': 1715715530889094,
'condition': 'host_os == "mac" and host_cpu == "x64"',
},
{
'object_name': 'Mac_arm64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '333dfb93cb0d944acb1ed9f0c44aba9dfdb4e7cd2c660758014338bea37ce8da',
'size_bytes': 135183068,
'generation': 1715715532458630,
'condition': 'host_os == "mac" and host_cpu == "arm64"',
},
{
'object_name': 'Win/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'f610825518d0f3b6f1e9d2772065165b5bcac3302448a53ac89b3684bae7dbd2',
'size_bytes': 206423884,
'generation': 1715715534082072,
'condition': 'host_os == "win"',
},
],
},
'src/third_party/clang-format/script':
Var('chromium_git') +
'/external/github.com/llvm/llvm-project/clang/tools/clang-format.git@' +
@ -4981,12 +5017,6 @@ hooks = [
'condition': 'not llvm_force_head_revision',
'action': ['vpython3', 'src/tools/clang/scripts/update.py'],
},
{
# Update prebuilt Rust toolchain.
'name': 'rust-toolchain',
'pattern': '.',
'action': ['vpython3', 'src/tools/rust/update_rust.py'],
},
{
# Build the clang toolchain from tip-of-tree.
# Note: On Win, this should run after win_toolchain, as it may use it.

3
docs/updating_clang.md

@ -26,6 +26,7 @@ An archive of all packages built so far is at https://is.gd/chromeclang
successfully finished, run
[go/chrome-promote-clang](https://goto.google.com/chrome-promote-clang)
on the new Clang package name.
1. Run `tools/clang/scripts/sync_deps.py` to update the deps entries in DEPS.
1. Run an exhaustive set of try jobs to test the new compiler. The CL
description created previously by upload_revision.py includes
`Cq-Include-Trybots:` lines for all needed bots, so it's sufficient to just
@ -83,4 +84,4 @@ tools/clang/scripts/update.py --package your-package-name
```
[an example of adding a new package](https://chromium-review.googlesource.com/c/chromium/src/+/5463029)
[example bug]: https://crbug.com/335730441
[example bug]: https://crbug.com/335730441

78
tools/clang/scripts/sync_deps.py Executable file

@ -0,0 +1,78 @@
#!/usr/bin/env python3
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script syncs the Clang and Rust revisions defined in update.py
and update_rust.py with the deps entries in DEPS."""
import argparse
import hashlib
import re
import os
import subprocess
import sys
import tempfile
from update import DownloadUrl, CDS_URL, CLANG_REVISION
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..',
'rust'))
from update_rust import RUST_REVISION, RUST_SUB_REVISION
def GetDepsObjectInfo(object_name: str) -> str:
url = f'{CDS_URL}/{object_name}'
describe_url = f'gs://chromium-browser-clang/{object_name}'
output = subprocess.check_output(['gsutil.py', 'stat',
describe_url]).decode("utf-8")
# Output looks like:
# ``
# gs://bucket/path:
# Creation time: Wed, 15 May 2024 13:36:30 GMT
# Update time: Wed, 15 May 2024 13:36:30 GMT
# Storage class: STANDARD
# Cache-Control: public, max-age=31536000,no-transform
# Content-Encoding: gzip
# Content-Length: 5766650
# Content-Type: application/octet-stream
# Hash (crc32c): E8z0Sg==
# Hash (md5): E/XAhJhhpd5+08cdO17CFA==
# ETag: COvj8aXjj4YDEAE=
# Generation: 1715780189975019
# Metageneration: 1
generation = re.search('Generation:\s+([0-9]+)', output).group(1)
size_bytes = re.search('Content-Length:\s+([0-9]+)', output).group(1)
with tempfile.NamedTemporaryFile() as f:
DownloadUrl(url, f)
f.seek(0)
sha256sum = hashlib.file_digest(f, 'sha256').hexdigest()
return f'{object_name},{sha256sum},{size_bytes},{generation}'
def GetRustObjectNames() -> list:
object_names = []
for host_os in ['Linux_x64', 'Mac', 'Mac_arm64', 'Win']:
rust_version = (f'{RUST_REVISION}-{RUST_SUB_REVISION}')
clang_revision = CLANG_REVISION
object_name = f'{host_os}/rust-toolchain-{rust_version}-{clang_revision}'
object_names.append(f'{object_name}.tar.xz')
return object_names
def main():
rust_object_infos = [
GetDepsObjectInfo(o) for o in sorted(GetRustObjectNames())
]
rust_object_infos_string = '?'.join(rust_object_infos)
rust_deps_entry_path = 'src/third_party/rust-toolchain'
rust_setdep_string = f'{rust_deps_entry_path}@{rust_object_infos_string}'
rust_setdep_args = ['gclient', 'setdep', f'--revision={rust_setdep_string}']
subprocess.run(rust_setdep_args)
if __name__ == '__main__':
sys.exit(main())

8
tools/rust/update_rust.py

@ -65,7 +65,6 @@ THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-toolchain')
# Path to the VERSION file stored in the archive.
VERSION_SRC_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'VERSION')
VERSION_STAMP_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'INSTALLED_VERSION')
def GetRustClangRevision():
@ -76,8 +75,8 @@ def GetRustClangRevision():
# Get the version of the toolchain package we already have.
def GetStampVersion():
if os.path.exists(VERSION_STAMP_PATH):
with open(VERSION_STAMP_PATH) as version_file:
if os.path.exists(VERSION_SRC_PATH):
with open(VERSION_SRC_PATH) as version_file:
existing_stamp = version_file.readline().rstrip()
version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)')
match = version_re.fullmatch(existing_stamp)
@ -142,9 +141,6 @@ def main():
try:
url = f'{platform_prefix}rust-toolchain-{version}.tar.xz'
DownloadAndUnpack(url, RUST_TOOLCHAIN_OUT_DIR)
# The archive contains a VERSION file. Copy it to INSTALLED_VERSION as
# the very last step in case the unpack fails after writing VERSION.
shutil.copyfile(VERSION_SRC_PATH, VERSION_STAMP_PATH)
except urllib.error.HTTPError as e:
print(f'error: Failed to download Rust package')
return 1