0

Reland "Reland "Reland Migrate to python 3.11 from 3.8""

This is a reland of commit a5a4a98193

The revert was caused by hanging swarming tasks which had a separate
version of python that wasn't synced to the buildbucket version.
(See crbug.com/395160553)

That version was fixed with https://chrome-internal-review.googlesource.com/c/infradata/config/+/8013788

and a bug has been filed for the versions not being in sync.

Original change's description:
> Reland "Reland Migrate to python 3.11 from 3.8"
>
> This is a reland of commit ae42a262c8
>
> The CL was reverted as it caused devices to hang when running tests
> on android. (https://github.com/python/cpython/issues/106883)
>
> This has been fixed with a patch that fixed a bug in the python
> 3.11 interpreter.
>
> https://chromium-review.googlesource.com/c/infra/infra/+/6155095
>
> And then rolling out that change to depot_tools:
> https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/6158273
>
> Original change's description:
> > Reland Migrate to python 3.11 from 3.8
> >
> > Original CL: https://chromium-review.googlesource.com/c/chromium/src/+/5872818
> > was reverted and was not able to create an actual reland because of
> > rebase merge failure with "create reland" button.
> >
> > Previous CL was reverted from failing:
> > chromeos-jacuzzi-rel
> > chromeos-octopus-rel
> >
> > This is from a incompatibility in aioquic, which is now fixed with
> > an update to aioquick 1.2 (and its supporting libraries) as well as
> > adding python and permission changes to chromeos.
> >
> > crrev.com/5904256
> > crrev.com/5904734
> >
> > and some internal changes.
> >
> > The webtransport_h3_server.py file has to be changed here with the
> > vpython change, and then wpt can be rolled. The change is already
> > in the wpt repo, but has been manually excluded in the roll.
> >
> > Bug: 40942322
> > Change-Id: Id12b7085fbc1ffd7694712013e6f746e65c3499f
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5887447
> > Reviewed-by: Ben Pastene <bpastene@chromium.org>
> > Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
> > Commit-Queue: Benjamin Joyce (Ben) <bjoyce@chromium.org>
> > Reviewed-by: Jonathan Lee <jonathanjlee@google.com>
> > Cr-Commit-Position: refs/heads/main@{#1370866}
>
> Bug: 40942322
> Change-Id: Icce4762639fa96104f81c114e853f8069c95cf85
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6168586
> Reviewed-by: Jonathan Lee <jonathanjlee@google.com>
> Reviewed-by: mmenke <mmenke@chromium.org>
> Reviewed-by: Ben Pastene <bpastene@chromium.org>
> Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
> Commit-Queue: Benjamin Joyce (Ben) <bjoyce@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#1410521}

Bug: 40942322
Change-Id: I10c428827d8c5a5618dd39f5a5f09484ee3f2d62
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6198919
Reviewed-by: mmenke <mmenke@chromium.org>
Reviewed-by: Ben Pastene <bpastene@chromium.org>
Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
Commit-Queue: Benjamin Joyce (Ben) <bjoyce@chromium.org>
Reviewed-by: Jonathan Lee <jonathanjlee@google.com>
Cr-Commit-Position: refs/heads/main@{#1420258}
This commit is contained in:
Ben Joyce
2025-02-13 18:34:07 -08:00
committed by Chromium LUCI CQ
parent a2aea5e344
commit 6f22ea472f
13 changed files with 93 additions and 143 deletions
.vpython3
infra/config
generated
builders
ci
chromeos-amd64-generic-rel-gtest
chromeos-amd64-generic-rel
try
chromeos-amd64-generic-rel-gtest-and-tast
chromeos-amd64-generic-rel-gtest
testing
targets
net/tools/testserver
third_party/wpt_tools
tools/perf

@ -22,20 +22,20 @@
# Read more about `vpython` and how to modify this file here:
# https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
python_version: "3.8"
python_version: "3.11"
# The default set of platforms vpython checks does not yet include mac-arm64.
# Setting `verify_pep425_tag` to the list of platforms we explicitly must support
# allows us to ensure that vpython specs stay mac-arm64-friendly
verify_pep425_tag: [
{python: "cp38", abi: "cp38", platform: "manylinux1_x86_64"},
{python: "cp38", abi: "cp38", platform: "linux_arm64"},
{python: "cp311", abi: "cp311", platform: "manylinux1_x86_64"},
{python: "cp311", abi: "cp311", platform: "linux_arm64"},
{python: "cp38", abi: "cp38", platform: "macosx_10_10_intel"},
{python: "cp38", abi: "cp38", platform: "macosx_11_0_arm64"},
{python: "cp311", abi: "cp311", platform: "macosx_10_10_intel"},
{python: "cp311", abi: "cp311", platform: "macosx_11_0_arm64"},
{python: "cp38", abi: "cp38", platform: "win32"},
{python: "cp38", abi: "cp38", platform: "win_amd64"}
{python: "cp311", abi: "cp311", platform: "win32"},
{python: "cp311", abi: "cp311", platform: "win_amd64"}
]
# Used by:
@ -111,7 +111,7 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/ninja/${vpython_platform}"
version: "version:1.10.0.post2"
version: "version:1.10.2.4.chromium.1"
match_tag: <
platform: "manylinux1_x86_64"
>
@ -150,7 +150,7 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/opencv_python/${vpython_platform}"
version: "version:4.5.3.56.chromium.4"
version: "version:4.8.1.78.chromium.1"
# There is currently no Linux arm/arm64 version in CIPD.
not_match_tag: <
platform: "linux_arm64"
@ -177,8 +177,8 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/pyfakefs-py2_py3"
version: "version:3.7.2"
name: "infra/python/wheels/pyfakefs-py3"
version: "version:5.5.0"
>
# Used by:
@ -189,7 +189,7 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/python-dateutil-py2_py3"
version: "version:2.7.3"
version: "version:2.8.1"
>
# Used by WPT importer
@ -210,8 +210,8 @@ wheel: <
version: "version:3.4.2"
>
wheel: <
name: "infra/python/wheels/cachetools-py2_py3"
version: "version:2.0.1"
name: "infra/python/wheels/cachetools-py3"
version: "version:5.3.3"
>
wheel: <
name: "infra/python/wheels/uritemplate-py3"
@ -255,7 +255,7 @@ wheel: <
# //third_party/chromite/bin/cros chrome-sdk
wheel: <
name: "infra/python/wheels/crcmod/${vpython_platform}"
version: "version:1.7.chromium.3"
version: "version:1.7.chromium.4"
>
wheel: <
@ -322,7 +322,7 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/mozprocess-py3"
version: "version:1.3.0"
version: "version:1.3.1"
>
wheel: <
name: "infra/python/wheels/urllib3-py2_py3"
@ -362,7 +362,7 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/aioquic/${vpython_platform}"
version: "version:0.9.20.chromium.1"
version: "version:1.2.0.chromium.1"
>
wheel: <
name: "infra/python/wheels/pylsqpack/${vpython_platform}"
@ -370,11 +370,11 @@ wheel: <
>
wheel: <
name: "infra/python/wheels/cryptography/${vpython_platform}"
version: "version:3.3.1.chromium.1"
version: "version:43.0.0"
>
wheel: <
name: "infra/python/wheels/cffi/${vpython_platform}"
version: "version:1.14.5.chromium.7"
version: "version:1.15.1.chromium.2"
>
wheel: <
name: "infra/python/wheels/pycparser-py2_py3"
@ -450,7 +450,7 @@ wheel: <
wheel: <
name: "infra/python/wheels/py-py2_py3"
version: "version:1.10.0"
version: "version:1.11.0"
>
wheel: <
@ -472,7 +472,7 @@ wheel: <
# //testing/buildbot/generate_buildbot_json_coveragetest.py
wheel: <
name: "infra/python/wheels/coverage/${vpython_platform}"
version: "version:5.5.chromium.3"
version: "version:7.3.1"
>
wheel: <
@ -494,7 +494,7 @@ wheel: <
# //tools/perf/crossbench
wheel: <
name: "infra/python/wheels/pytz-py2_py3"
version: "version:2018.4"
version: "version:2024.1"
>
# Used by:
@ -514,7 +514,7 @@ wheel: <
# //content/test/gpu/gpu_tests/color_profile_manager_mac.py
wheel: <
name: "infra/python/wheels/pyobjc/${vpython_platform}"
version: "version:7.3.chromium.1"
version: "version:10.0"
match_tag: <
platform: "macosx_10_10_intel"
>
@ -535,7 +535,7 @@ wheel: <
# //tools/perf/crossbench
wheel: <
name: "infra/python/wheels/pandas/${vpython_platform}"
version: "version:1.3.2.chromium.1"
version: "version:1.5.3.chromium.1"
not_match_tag: <
platform: "linux_arm64"
>
@ -573,6 +573,10 @@ wheel: <
name: "infra/python/wheels/selenium-py3"
version: "version:4.1.0"
>
wheel: <
name: "infra/python/wheels/service-identity-py3"
version: "version:24.1.0"
>
wheel: <
name: "infra/python/wheels/trio-py3"
version: "version:0.20.0"
@ -606,8 +610,8 @@ wheel: <
version: "version:2.4.0"
>
wheel: <
name: "infra/python/wheels/pyopenssl-py2_py3"
version: "version:20.0.0"
name: "infra/python/wheels/pyopenssl-py3"
version: "version:24.2.1"
>
# Used by //tools/md_browser to render Markdown.
@ -682,6 +686,12 @@ wheel: <
version: "version:3.1.2"
>
# Needed by pytype
wheel: <
name: "infra/python/wheels/pysocks-py3"
version: "version:1.7.1"
>
# Needed by pytype
wheel: <
name: "infra/python/wheels/pycnite-py3"
@ -700,3 +710,7 @@ wheel: <
version: "version:2.0.1"
>
wheel: <
name: "infra/python/wheels/apipkg-py2_py3"
version: "version:1.5"
>

@ -492,7 +492,7 @@
{
"cipd_package": "infra/3pp/tools/cpython3/linux-amd64",
"location": "vpython_dir_linux_amd64",
"revision": "version:2@3.8.10.chromium.34"
"revision": "version:3@3.11.9.chromium.36"
},
{
"cipd_package": "infra/tools/luci/vpython3/linux-amd64",

@ -497,7 +497,7 @@
{
"cipd_package": "infra/3pp/tools/cpython3/linux-amd64",
"location": "vpython_dir_linux_amd64",
"revision": "version:2@3.8.10.chromium.34"
"revision": "version:3@3.11.9.chromium.36"
},
{
"cipd_package": "infra/tools/luci/vpython3/linux-amd64",

@ -497,7 +497,7 @@
{
"cipd_package": "infra/3pp/tools/cpython3/linux-amd64",
"location": "vpython_dir_linux_amd64",
"revision": "version:2@3.8.10.chromium.34"
"revision": "version:3@3.11.9.chromium.36"
},
{
"cipd_package": "infra/tools/luci/vpython3/linux-amd64",

@ -497,7 +497,7 @@
{
"cipd_package": "infra/3pp/tools/cpython3/linux-amd64",
"location": "vpython_dir_linux_amd64",
"revision": "version:2@3.8.10.chromium.34"
"revision": "version:3@3.11.9.chromium.36"
},
{
"cipd_package": "infra/tools/luci/vpython3/linux-amd64",

@ -248,7 +248,7 @@
{
'cipd_package': 'infra/3pp/tools/cpython3/linux-amd64',
'location': 'vpython_dir_linux_amd64',
'revision': 'version:2@3.8.10.chromium.34',
'revision': 'version:3@3.11.9.chromium.36',
},
{
'cipd_package': 'infra/tools/luci/vpython3/linux-amd64',

@ -228,7 +228,7 @@ targets.legacy_basic_suite(
targets.cipd_package(
package = "infra/3pp/tools/cpython3/linux-amd64",
location = "vpython_dir_linux_amd64",
revision = "version:2@3.8.10.chromium.34",
revision = "version:3@3.11.9.chromium.36",
),
targets.cipd_package(
package = "infra/tools/luci/vpython3/linux-amd64",

@ -26,20 +26,20 @@
# many dependencies and trying to load all dependencies from the main .vpython3
# spec on ChromeOS VMs runs into disk space issues.
python_version: "3.8"
python_version: "3.11"
# The default set of platforms vpython checks does not yet include mac-arm64.
# Setting `verify_pep425_tag` to the list of platforms we explicitly must support
# allows us to ensure that vpython specs stay mac-arm64-friendly
verify_pep425_tag: [
{python: "cp38", abi: "cp38", platform: "manylinux1_x86_64"},
{python: "cp38", abi: "cp38", platform: "linux_arm64"},
{python: "cp38", abi: "cp311", platform: "manylinux1_x86_64"},
{python: "cp38", abi: "cp311", platform: "linux_arm64"},
{python: "cp38", abi: "cp38", platform: "macosx_10_10_intel"},
{python: "cp38", abi: "cp38", platform: "macosx_11_0_arm64"},
{python: "cp38", abi: "cp311", platform: "macosx_10_10_intel"},
{python: "cp38", abi: "cp311", platform: "macosx_11_0_arm64"},
{python: "cp38", abi: "cp38", platform: "win32"},
{python: "cp38", abi: "cp38", platform: "win_amd64"}
{python: "cp38", abi: "cp311", platform: "win32"},
{python: "cp38", abi: "cp311", platform: "win_amd64"}
]
wheel: <

@ -13,6 +13,3 @@ Description: This includes code for the manifest tool, lint tool, and wptserve.
for more details on maintenance.
Local Modifications:
- Removed all files except for those listed in wpt/WPTIncludeList.
- Local patch to the webtransport-H3 server to stay on `aioquic` 0.x, which
can't be upgraded to 1.x until chromium/src is on python3.11
(https://crbug.com/387535231).

@ -1,90 +0,0 @@
diff --git a/tools/webtransport/h3/webtransport_h3_server.py b/tools/webtransport/h3/webtransport_h3_server.py
index 2dd8f645551d6..8c55ba87e0e59 100644
--- a/tools/webtransport/h3/webtransport_h3_server.py
+++ b/tools/webtransport/h3/webtransport_h3_server.py
@@ -1,7 +1,6 @@
# mypy: allow-subclassing-any, no-warn-return-any
import asyncio
-import contextlib
import logging
import os
import ssl
@@ -10,21 +9,17 @@ import threading
import traceback
from enum import IntEnum
from urllib.parse import urlparse
-from typing import Any, Dict, List, Optional, Tuple, cast
+from typing import Any, Dict, List, Optional, Tuple
# TODO(bashi): Remove import check suppressions once aioquic dependency is resolved.
from aioquic.buffer import Buffer # type: ignore
from aioquic.asyncio import QuicConnectionProtocol, serve # type: ignore
from aioquic.asyncio.client import connect # type: ignore
-from aioquic.asyncio.protocol import QuicStreamAdapter # type: ignore
from aioquic.h3.connection import H3_ALPN, FrameType, H3Connection, ProtocolError, SettingsError # type: ignore
from aioquic.h3.events import H3Event, HeadersReceived, WebTransportStreamDataReceived, DatagramReceived, DataReceived # type: ignore
from aioquic.quic.configuration import QuicConfiguration # type: ignore
from aioquic.quic.connection import logger as quic_connection_logger # type: ignore
-from aioquic.quic.connection import (
- stream_is_client_initiated,
- stream_is_unidirectional,
-)
+from aioquic.quic.connection import stream_is_unidirectional
from aioquic.quic.events import QuicEvent, ProtocolNegotiated, ConnectionTerminated, StreamReset # type: ignore
from aioquic.tls import SessionTicket # type: ignore
@@ -415,7 +410,7 @@ class WebTransportSession:
_logger.warn(
"Sending a datagram while that's now allowed - discarding it")
return
- stream_id = self.session_id
+ flow_id = self.session_id
if self._http.datagram_setting is not None:
# We must have a WebTransport Session ID at this point because
# an extended CONNECT request is already received.
@@ -423,8 +418,8 @@ class WebTransportSession:
# TODO(yutakahirano): Make sure if this is the correct logic.
# Chrome always use 0 for the initial stream and the initial flow
# ID, we cannot check the correctness with it.
- stream_id = self._protocol._session_stream_id
- self._http.send_datagram(stream_id=stream_id, data=data)
+ flow_id = self._protocol._session_stream_id // 4
+ self._http.send_datagram(flow_id=flow_id, data=data)
def stop_stream(self, stream_id: int, code: int) -> None:
"""
@@ -607,25 +602,6 @@ async def _connect_server_with_timeout(host: str, port: int, timeout: float) ->
return True
-def _close_unusable_writer(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
- # Starting in python3.11, `StreamWriter.__del__` implicitly `close()`s
- # itself [0], if it has not done so already. Because aioquic sometimes
- # models a unidirectional stream with a bidirectional transport [1], the
- # `writer` here for a server -> client stream may log a benign but
- # scary-looking exception when it's GC'ed and unsuccessfully writes EOF.
- #
- # For the purpose of checking connectivity, work around this for now by
- # preemptively closing such streams and suppressing the resulting
- # exceptions.
- #
- # [0]: https://github.com/python/cpython/blob/3.11/Lib/asyncio/streams.py#L413
- # [1]: https://github.com/aiortc/aioquic/blob/1.2.0/src/aioquic/asyncio/protocol.py#L241
- stream_id = cast(QuicStreamAdapter, writer.transport).stream_id
- if stream_is_unidirectional(stream_id) and not stream_is_client_initiated(stream_id):
- with contextlib.suppress(ValueError):
- writer.close()
-
-
async def _connect_to_server(host: str, port: int) -> None:
configuration = QuicConfiguration(
alpn_protocols=H3_ALPN,
@@ -633,6 +609,5 @@ async def _connect_to_server(host: str, port: int) -> None:
verify_mode=ssl.CERT_NONE,
)
- async with connect(host, port, configuration=configuration,
- stream_handler=_close_unusable_writer) as protocol:
+ async with connect(host, port, configuration=configuration) as protocol:
await protocol.ping()

@ -1,6 +1,7 @@
# mypy: allow-subclassing-any, no-warn-return-any
import asyncio
import contextlib
import logging
import os
import ssl
@ -9,17 +10,21 @@ import threading
import traceback
from enum import IntEnum
from urllib.parse import urlparse
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple, cast
# TODO(bashi): Remove import check suppressions once aioquic dependency is resolved.
from aioquic.buffer import Buffer # type: ignore
from aioquic.asyncio import QuicConnectionProtocol, serve # type: ignore
from aioquic.asyncio.client import connect # type: ignore
from aioquic.asyncio.protocol import QuicStreamAdapter # type: ignore
from aioquic.h3.connection import H3_ALPN, FrameType, H3Connection, ProtocolError, SettingsError # type: ignore
from aioquic.h3.events import H3Event, HeadersReceived, WebTransportStreamDataReceived, DatagramReceived, DataReceived # type: ignore
from aioquic.quic.configuration import QuicConfiguration # type: ignore
from aioquic.quic.connection import logger as quic_connection_logger # type: ignore
from aioquic.quic.connection import stream_is_unidirectional
from aioquic.quic.connection import (
stream_is_client_initiated,
stream_is_unidirectional,
)
from aioquic.quic.events import QuicEvent, ProtocolNegotiated, ConnectionTerminated, StreamReset # type: ignore
from aioquic.tls import SessionTicket # type: ignore
@ -410,7 +415,7 @@ class WebTransportSession:
_logger.warn(
"Sending a datagram while that's now allowed - discarding it")
return
flow_id = self.session_id
stream_id = self.session_id
if self._http.datagram_setting is not None:
# We must have a WebTransport Session ID at this point because
# an extended CONNECT request is already received.
@ -418,8 +423,8 @@ class WebTransportSession:
# TODO(yutakahirano): Make sure if this is the correct logic.
# Chrome always use 0 for the initial stream and the initial flow
# ID, we cannot check the correctness with it.
flow_id = self._protocol._session_stream_id // 4
self._http.send_datagram(flow_id=flow_id, data=data)
stream_id = self._protocol._session_stream_id
self._http.send_datagram(stream_id=stream_id, data=data)
def stop_stream(self, stream_id: int, code: int) -> None:
"""
@ -602,6 +607,25 @@ async def _connect_server_with_timeout(host: str, port: int, timeout: float) ->
return True
def _close_unusable_writer(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
# Starting in python3.11, `StreamWriter.__del__` implicitly `close()`s
# itself [0], if it has not done so already. Because aioquic sometimes
# models a unidirectional stream with a bidirectional transport [1], the
# `writer` here for a server -> client stream may log a benign but
# scary-looking exception when it's GC'ed and unsuccessfully writes EOF.
#
# For the purpose of checking connectivity, work around this for now by
# preemptively closing such streams and suppressing the resulting
# exceptions.
#
# [0]: https://github.com/python/cpython/blob/3.11/Lib/asyncio/streams.py#L413
# [1]: https://github.com/aiortc/aioquic/blob/1.2.0/src/aioquic/asyncio/protocol.py#L241
stream_id = cast(QuicStreamAdapter, writer.transport).stream_id
if stream_is_unidirectional(stream_id) and not stream_is_client_initiated(stream_id):
with contextlib.suppress(ValueError):
writer.close()
async def _connect_to_server(host: str, port: int) -> None:
configuration = QuicConfiguration(
alpn_protocols=H3_ALPN,
@ -609,5 +633,6 @@ async def _connect_to_server(host: str, port: int) -> None:
verify_mode=ssl.CERT_NONE,
)
async with connect(host, port, configuration=configuration) as protocol:
async with connect(host, port, configuration=configuration,
stream_handler=_close_unusable_writer) as protocol:
await protocol.ping()

@ -23,12 +23,16 @@ class FindDependenciesTest(unittest.TestCase):
'dependency_test_dir', 'other_animals', 'cat', 'cat')
cat_module_init_path = os.path.join(cat_module_path, '__init__.py')
cat_object_path = os.path.join(cat_module_path, 'cat_object.py')
dependencies = set(
p for p in find_dependencies.FindPythonDependencies(dog_object_path))
self.assertEqual(dependencies, {
answer_set = {
dog_object_path, cat_module_path, cat_module_init_path,
cat_object_path
})
}
dependencies = set(
p for p in find_dependencies.FindPythonDependencies(dog_object_path))
for path in answer_set:
self.assertIn(path, dependencies)
except ImportError: # crbug.com/559527
pass

@ -83,7 +83,7 @@ class ScriptsSmokeTest(unittest.TestCase):
def testRunRecordWprHelp(self):
return_code, stdout = self.RunPerfScript('record_wpr')
self.assertEqual(return_code, 0, stdout)
self.assertIn('optional arguments:', stdout)
self.assertIn('positional arguments:', stdout)
@decorators.Disabled('chromeos') # crbug.com/814068
def testRunRecordWprList(self):