0

Reland "Migrate rust hook to deps and create sync_deps.py"

This is a reland of commit 6f00b586dd

Original change's description:
> Reland "Migrate rust hook to deps and create sync_deps.py"
>
> This is a reland of commit 2af7751ec4, with the
> following changes:
> * update_rust - use VERSION instead of INSTALLED_VERSION
>
> gclient ensures that the rust package is not partially extracted, so
> INSTALLED_VERSION is no longer needed.
>
>
> Original change's description:
> > Migrate rust hook to deps and create sync_deps.py
> >
> > After someone runs upload_revision.py, triggers the packager bots,
> > and manually copies the packages to prod, they will now run
> > sync_deps.py to update the DEPS file. After doing that they can
> > trigger a CQ run to test the new packages.
> >
> > Bug: b/338612245
> > Change-Id: Ie95efbf2049e651192d52ca7531c20125b3dd537
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5536220
> > Reviewed-by: Hans Wennborg <hans@chromium.org>
> > Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
> > Cr-Commit-Position: refs/heads/main@{#1304616}
>
> Bug: b/338612245
> Change-Id: I2038e57a2e767eea10df6e2b72b6d2e2344d630e
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5560825
> Reviewed-by: Hans Wennborg <hans@chromium.org>
> Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
> Reviewed-by: danakj <danakj@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#1305165}

Bug: b/338612245
Change-Id: I597a7ee0e894109e34cd374bf20d270a01705354
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5569382
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Hans Wennborg <hans@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1308016}
This commit is contained in:
Josip Sokcevic
2024-05-30 15:25:38 +00:00
committed by Chromium LUCI CQ
parent 08dcbd1413
commit a41c3c45fd
4 changed files with 117 additions and 12 deletions

42
DEPS

@@ -543,6 +543,7 @@ allowed_hosts = [
# TODO(337061377): Move into a separate allowed gcs bucket list. # TODO(337061377): Move into a separate allowed gcs bucket list.
'chromium-ads-detection', 'chromium-ads-detection',
'chromium-browser-clang',
'chromium-clang-format', 'chromium-clang-format',
'chromium-nodejs', 'chromium-nodejs',
'chromium-tfhub-models', 'chromium-tfhub-models',
@@ -780,6 +781,41 @@ deps = {
}, },
], ],
}, },
# Update prebuilt Rust toolchain.
'src/third_party/rust-toolchain': {
'dep_type': 'gcs',
'bucket': 'chromium-browser-clang',
'objects': [
{
'object_name': 'Linux_x64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'e7e236fe6e61b7a41ba6b1e4bc64e91119359cdddce268597c6328b9fea8d76d',
'size_bytes': 151082700,
'generation': 1715715529295267,
'condition': 'host_os == "linux" and non_git_source',
},
{
'object_name': 'Mac/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '3860d592e7f6a853aa06a00642f4110132c7af7f2b91d7bdd399d00dbd5177f6',
'size_bytes': 143827372,
'generation': 1715715530889094,
'condition': 'host_os == "mac" and host_cpu == "x64"',
},
{
'object_name': 'Mac_arm64/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': '333dfb93cb0d944acb1ed9f0c44aba9dfdb4e7cd2c660758014338bea37ce8da',
'size_bytes': 135183068,
'generation': 1715715532458630,
'condition': 'host_os == "mac" and host_cpu == "arm64"',
},
{
'object_name': 'Win/rust-toolchain-32dd3795bce8b347fda786529cf5e42a813e0b7d-2-llvmorg-19-init-10646-g084e2b53.tar.xz',
'sha256sum': 'f610825518d0f3b6f1e9d2772065165b5bcac3302448a53ac89b3684bae7dbd2',
'size_bytes': 206423884,
'generation': 1715715534082072,
'condition': 'host_os == "win"',
},
],
},
'src/third_party/clang-format/script': 'src/third_party/clang-format/script':
Var('chromium_git') + Var('chromium_git') +
'/external/github.com/llvm/llvm-project/clang/tools/clang-format.git@' + '/external/github.com/llvm/llvm-project/clang/tools/clang-format.git@' +
@@ -5057,12 +5093,6 @@ hooks = [
'condition': 'not llvm_force_head_revision', 'condition': 'not llvm_force_head_revision',
'action': ['vpython3', 'src/tools/clang/scripts/update.py'], 'action': ['vpython3', 'src/tools/clang/scripts/update.py'],
}, },
{
# Update prebuilt Rust toolchain.
'name': 'rust-toolchain',
'pattern': '.',
'action': ['vpython3', 'src/tools/rust/update_rust.py'],
},
{ {
# Build the clang toolchain from tip-of-tree. # Build the clang toolchain from tip-of-tree.
# Note: On Win, this should run after win_toolchain, as it may use it. # Note: On Win, this should run after win_toolchain, as it may use it.

@@ -26,6 +26,7 @@ An archive of all packages built so far is at https://is.gd/chromeclang
successfully finished, run successfully finished, run
[go/chrome-promote-clang](https://goto.google.com/chrome-promote-clang) [go/chrome-promote-clang](https://goto.google.com/chrome-promote-clang)
on the new Clang package name. on the new Clang package name.
1. Run `tools/clang/scripts/sync_deps.py` to update the deps entries in DEPS.
1. Run an exhaustive set of try jobs to test the new compiler. The CL 1. Run an exhaustive set of try jobs to test the new compiler. The CL
description created previously by upload_revision.py includes description created previously by upload_revision.py includes
`Cq-Include-Trybots:` lines for all needed bots, so it's sufficient to just `Cq-Include-Trybots:` lines for all needed bots, so it's sufficient to just

@@ -0,0 +1,78 @@
#!/usr/bin/env python3
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script syncs the Clang and Rust revisions defined in update.py
and update_rust.py with the deps entries in DEPS."""
import argparse
import hashlib
import re
import os
import subprocess
import sys
import tempfile
from update import DownloadUrl, CDS_URL, CLANG_REVISION
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..',
'rust'))
from update_rust import RUST_REVISION, RUST_SUB_REVISION
def GetDepsObjectInfo(object_name: str) -> str:
url = f'{CDS_URL}/{object_name}'
describe_url = f'gs://chromium-browser-clang/{object_name}'
output = subprocess.check_output(['gsutil.py', 'stat',
describe_url]).decode("utf-8")
# Output looks like:
# ``
# gs://bucket/path:
# Creation time: Wed, 15 May 2024 13:36:30 GMT
# Update time: Wed, 15 May 2024 13:36:30 GMT
# Storage class: STANDARD
# Cache-Control: public, max-age=31536000,no-transform
# Content-Encoding: gzip
# Content-Length: 5766650
# Content-Type: application/octet-stream
# Hash (crc32c): E8z0Sg==
# Hash (md5): E/XAhJhhpd5+08cdO17CFA==
# ETag: COvj8aXjj4YDEAE=
# Generation: 1715780189975019
# Metageneration: 1
generation = re.search('Generation:\s+([0-9]+)', output).group(1)
size_bytes = re.search('Content-Length:\s+([0-9]+)', output).group(1)
with tempfile.NamedTemporaryFile() as f:
DownloadUrl(url, f)
f.seek(0)
sha256sum = hashlib.file_digest(f, 'sha256').hexdigest()
return f'{object_name},{sha256sum},{size_bytes},{generation}'
def GetRustObjectNames() -> list:
object_names = []
for host_os in ['Linux_x64', 'Mac', 'Mac_arm64', 'Win']:
rust_version = (f'{RUST_REVISION}-{RUST_SUB_REVISION}')
clang_revision = CLANG_REVISION
object_name = f'{host_os}/rust-toolchain-{rust_version}-{clang_revision}'
object_names.append(f'{object_name}.tar.xz')
return object_names
def main():
rust_object_infos = [
GetDepsObjectInfo(o) for o in sorted(GetRustObjectNames())
]
rust_object_infos_string = '?'.join(rust_object_infos)
rust_deps_entry_path = 'src/third_party/rust-toolchain'
rust_setdep_string = f'{rust_deps_entry_path}@{rust_object_infos_string}'
rust_setdep_args = ['gclient', 'setdep', f'--revision={rust_setdep_string}']
subprocess.run(rust_setdep_args)
if __name__ == '__main__':
sys.exit(main())

@@ -65,7 +65,6 @@ THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-toolchain') RUST_TOOLCHAIN_OUT_DIR = os.path.join(THIRD_PARTY_DIR, 'rust-toolchain')
# Path to the VERSION file stored in the archive. # Path to the VERSION file stored in the archive.
VERSION_SRC_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'VERSION') VERSION_SRC_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'VERSION')
VERSION_STAMP_PATH = os.path.join(RUST_TOOLCHAIN_OUT_DIR, 'INSTALLED_VERSION')
def GetRustClangRevision(): def GetRustClangRevision():
@@ -76,8 +75,8 @@ def GetRustClangRevision():
# Get the version of the toolchain package we already have. # Get the version of the toolchain package we already have.
def GetStampVersion(): def GetStampVersion():
if os.path.exists(VERSION_STAMP_PATH): if os.path.exists(VERSION_SRC_PATH):
with open(VERSION_STAMP_PATH) as version_file: with open(VERSION_SRC_PATH) as version_file:
existing_stamp = version_file.readline().rstrip() existing_stamp = version_file.readline().rstrip()
version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)') version_re = re.compile(r'rustc [0-9.]+ [0-9a-f]+ \((.+?) chromium\)')
match = version_re.fullmatch(existing_stamp) match = version_re.fullmatch(existing_stamp)
@@ -142,9 +141,6 @@ def main():
try: try:
url = f'{platform_prefix}rust-toolchain-{version}.tar.xz' url = f'{platform_prefix}rust-toolchain-{version}.tar.xz'
DownloadAndUnpack(url, RUST_TOOLCHAIN_OUT_DIR) DownloadAndUnpack(url, RUST_TOOLCHAIN_OUT_DIR)
# The archive contains a VERSION file. Copy it to INSTALLED_VERSION as
# the very last step in case the unpack fails after writing VERSION.
shutil.copyfile(VERSION_SRC_PATH, VERSION_STAMP_PATH)
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
print(f'error: Failed to download Rust package') print(f'error: Failed to download Rust package')
return 1 return 1