0

Reland "Migrate rust hook to deps and create sync_deps.py"

This is a reland of commit 2af7751ec4, with the
following changes:
* update_rust - use VERSION instead of INSTALLED_VERSION

gclient ensures that the rust package is not partially extracted, so
INSTALLED_VERSION is no longer needed.


Original change's description:
> Migrate rust hook to deps and create sync_deps.py
>
> After someone runs upload_revision.py, triggers the packager bots,
> and manually copies the packages to prod, they will now run
> sync_deps.py to update the DEPS file. After doing that they can
> trigger a CQ run to test the new packages.
>
> Bug: b/338612245
> Change-Id: Ie95efbf2049e651192d52ca7531c20125b3dd537
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5536220
> Reviewed-by: Hans Wennborg <hans@chromium.org>
> Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#1304616}

Bug: b/338612245
Change-Id: I2038e57a2e767eea10df6e2b72b6d2e2344d630e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5560825
Reviewed-by: Hans Wennborg <hans@chromium.org>
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: danakj <danakj@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1305165}
This commit is contained in:
Josip Sokcevic
2024-05-23 17:02:19 +00:00
committed by Chromium LUCI CQ
parent 5c897e0ed5
commit 6f00b586dd
4 changed files with 118 additions and 13 deletions

42
DEPS

@@ -540,6 +540,7 @@ allowed_hosts = [
# TODO(337061377): Move into a separate allowed gcs bucket list. # TODO(337061377): Move into a separate allowed gcs bucket list.
'chromium-ads-detection', 'chromium-ads-detection',
'chromium-browser-clang',
'chromium-clang-format', 'chromium-clang-format',
'chromium-nodejs', 'chromium-nodejs',
'chromium-tfhub-models', 'chromium-tfhub-models',
@@ -713,6 +714,41 @@ deps = {
}, },
], ],
}, },
# Update prebuilt Rust toolchain.
'src/third_party/rust-toolchain': {
'dep_type': 'gcs',
'bucket': 'chromium-browser-clang',
'objects': [
{
'object_name': 'Linux_x64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'e7e236fe6e61b7a41ba6b1e4bc64e91119359cdddce268597c6328b9fea8d76d',
'size_bytes': 151082700,
'generation': 1715715529295267,
'condition': 'host_os == "linux" and non_git_source',
},
{
'object_name': 'Mac/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '3860d592e7f6a853aa06a00642f4110132c7af7f2b91d7bdd399d00dbd5177f6',
'size_bytes': 143827372,
'generation': 1715715530889094,
'condition': 'host_os == "mac" and host_cpu == "x64"',
},
{
'object_name': 'Mac_arm64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '333dfb93cb0d944acb1ed9f0c44aba9dfdb4e7cd2c660758014338bea37ce8da',
'size_bytes': 135183068,
'generation': 1715715532458630,
'condition': 'host_os == "mac" and host_cpu == "arm64"',
},
{
'object_name': 'Win/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'f610825518d0f3b6f1e9d2772065165b5bcac3302448a53ac89b3684bae7dbd2',
'size_bytes': 206423884,
'generation': 1715715534082072,
'condition': 'host_os == "win"',
},
],
},
'src/third_party/clang-format/script': 'src/third_party/clang-format/script':
Var('chromium_git') + Var('chromium_git') +
'/external/github.com/llvm/llvm-project/clang/tools/clang-format.git@' + '/external/github.com/llvm/llvm-project/clang/tools/clang-format.git@' +
@@ -4981,12 +5017,6 @@ hooks = [
'condition': 'not llvm_force_head_revision', 'condition': 'not llvm_force_head_revision',
'action': ['vpython3', 'src/tools/clang/scripts/update.py'], 'action': ['vpython3', 'src/tools/clang/scripts/update.py'],
}, },
{
# Update prebuilt Rust toolchain.
'name': 'rust-toolchain',
'pattern': '.',
'action': ['vpython3', 'src/tools/rust/update_rust.py'],
},
{ {
# Build the clang toolchain from tip-of-tree. # Build the clang toolchain from tip-of-tree.
# Note: On Win, this should run after win_toolchain, as it may use it. # Note: On Win, this should run after win_toolchain, as it may use it.

@@ -26,6 +26,7 @@ An archive of all packages built so far is at https://is.gd/chromeclang
successfully finished, run successfully finished, run
[go/chrome-promote-clang](https://goto.google.com/chrome-promote-clang) [go/chrome-promote-clang](https://goto.google.com/chrome-promote-clang)
on the new Clang package name. on the new Clang package name.
1. Run `tools/clang/scripts/sync_deps.py` to update the deps entries in DEPS.
1. Run an exhaustive set of try jobs to test the new compiler. The CL 1. Run an exhaustive set of try jobs to test the new compiler. The CL
description created previously by upload_revision.py includes description created previously by upload_revision.py includes
`Cq-Include-Trybots:` lines for all needed bots, so it's sufficient to just `Cq-Include-Trybots:` lines for all needed bots, so it's sufficient to just
@@ -83,4 +84,4 @@ tools/clang/scripts/update.py --package your-package-name
``` ```
[an example of adding a new package](https://chromium-review.googlesource.com/c/chromium/src/+/5463029) [an example of adding a new package](https://chromium-review.googlesource.com/c/chromium/src/+/5463029)
[example bug]: https://crbug.com/335730441 [example bug]: https://crbug.com/335730441

@@ -0,0 +1,78 @@
#!/usr/bin/env python3
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script syncs the Clang and Rust revisions defined in update.py
and update_rust.py with the deps entries in DEPS."""
import argparse
import hashlib
import re
import os
import subprocess
import sys
import tempfile
from update import DownloadUrl, CDS_URL, CLANG_REVISION
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..',
'rust'))
from update_rust import RUST_REVISION, RUST_SUB_REVISION
def GetDepsObjectInfo(object_name: str) -> str:
url = f'{CDS_URL}/{object_name}'
describe_url = f'gs://chromium-browser-clang/{object_name}'
output = subprocess.check_output(['gsutil.py', 'stat',
describe_url]).decode("utf-8")
# Output looks like:
# ``
# gs://bucket/path:
# Creation time: Wed, 15 May 2024 13:36:30 GMT
# Update time: Wed, 15 May 2024 13:36:30 GMT
# Storage class: STANDARD
# Cache-Control: public, max-age=31536000,no-transform
# Content-Encoding: gzip
# Content-Length: 5766650
# Content-Type: application/octet-stream
# Hash (crc32c): E8z0Sg==
# Hash (md5): E/XAhJhhpd5+08cdO17CFA==
# ETag: COvj8aXjj4YDEAE=
# Generation: 1715780189975019
# Metageneration: 1
generation = re.search('Generation:\s+([0-9]+)', output).group(1)
size_bytes = re.search('Content-Length:\s+([0-9]+)', output).group(1)
with tempfile.NamedTemporaryFile() as f:
DownloadUrl(url, f)
f.seek(0)
sha256sum = hashlib.file_digest(f, 'sha256').hexdigest()
return f'{object_name},{sha256sum},{size_bytes},{generation}'
def GetRustObjectNames() -> list:
object_names = []
for host_os in ['Linux_x64', 'Mac', 'Mac_arm64', 'Win']:
rust_version = (f'{RUST_REVISION}-{RUST_SUB_REVISION}')
clang_revision = CLANG_REVISION
object_name = f'{host_os}/rust-toolchain-{rust_version}-{clang_revision}'
object_names.append(f'{object_name}.tar.xz')
return object_names
def main():
rust_object_infos = [
GetDepsObjectInfo(o) for o in sorted(GetRustObjectNames())
]
rust_object_infos_string = '?'.join(rust_object_infos)
rust_deps_entry_path = 'src/third_party/rust-toolchain'
rust_setdep_string = f'{rust_deps_entry_path}@{rust_object_infos_string}'
rust_setdep_args = ['gclient', 'setdep', f'--revision={rust_setdep_string}']
subprocess.run(rust_setdep_args)
if __name__ == '__main__':
sys.exit(main())

@@ -65,7 +65,6 @@ THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-toolchain') RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-toolchain')
# Path to the VERSION file stored in the archive. # Path to the VERSION file stored in the archive.
VERSION_SRC_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'VERSION') VERSION_SRC_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'VERSION')
VERSION_STAMP_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'INSTALLED_VERSION')
def GetRustClangRevision(): def GetRustClangRevision():
@@ -76,8 +75,8 @@ def GetRustClangRevision():
# Get the version of the toolchain package we already have. # Get the version of the toolchain package we already have.
def GetStampVersion(): def GetStampVersion():
if os.path.exists(VERSION_STAMP_PATH): if os.path.exists(VERSION_SRC_PATH):
with open(VERSION_STAMP_PATH) as version_file: with open(VERSION_SRC_PATH) as version_file:
existing_stamp = version_file.readline().rstrip() existing_stamp = version_file.readline().rstrip()
version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)') version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)')
match = version_re.fullmatch(existing_stamp) match = version_re.fullmatch(existing_stamp)
@@ -142,9 +141,6 @@ def main():
try: try:
url = f'{platform_prefix}rust-toolchain-{version}.tar.xz' url = f'{platform_prefix}rust-toolchain-{version}.tar.xz'
DownloadAndUnpack(url, RUST_TOOLCHAIN_OUT_DIR) DownloadAndUnpack(url, RUST_TOOLCHAIN_OUT_DIR)
# The archive contains a VERSION file. Copy it to INSTALLED_VERSION as
# the very last step in case the unpack fails after writing VERSION.
shutil.copyfile(VERSION_SRC_PATH, VERSION_STAMP_PATH)
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
print(f'error: Failed to download Rust package') print(f'error: Failed to download Rust package')
return 1 return 1