0

[fuzzing] Add high-end ASAN centipede builder with dchecks

This adds a centipede builder that only builds the V8 fuzzer
targets, which require dchecks to find the most useful bugs.

This is similar to https://crrev.com/c/6434210 but with centipede
for the high-end builder config. Centipede has a much better
performance as libFuzzer according to the stats.

After a grace period, we'll disable the V8 targets on the
high-end release builder without dchecks.

By only building V8 targets, this builder will be quite fast and
won't have a long cycle time.

Bug: 408843424
Change-Id: I9c421c2ef6b8295a31cb035adf3fd3e96057f966
Cq-Include-Trybots: luci.chromium.try:linux-centipede-asan-rel
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6540831
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Paul Semel <paulsemel@chromium.org>
Reviewed-by: Ben Pastene <bpastene@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1459930}
This commit is contained in:
Michael Achenbach
2025-05-14 01:27:08 -07:00
committed by Chromium LUCI CQ
parent 01895cede7
commit 85d03692df
17 changed files with 562 additions and 8 deletions

@ -1,4 +1,5 @@
ci/Centipede High End Upload Linux ASan
ci/Centipede High End Upload Linux ASan DCheck
ci/Centipede Upload Linux ASan
ci/Libfuzzer High End Upload Linux ASan
ci/Libfuzzer High End Upload Linux ASan Debug
@ -12,6 +13,7 @@ ci/Libfuzzer Upload Mac ASan
ci/Libfuzzer Upload Windows ASan
ci/Libfuzzer Upload iOS Catalyst Debug
try/chromeos-libfuzzer-asan-rel
try/linux-centipede-high-end-asan-dcheck
try/linux-libfuzzer-high-end-asan-dbg
try/linux-libfuzzer-high-end-asan-rel
try/mac-libfuzzer-asan-rel

@ -0,0 +1,17 @@
{
"gn_args": {
"archive_seed_corpus": false,
"dcheck_always_on": true,
"high_end_fuzzer_targets": true,
"is_asan": true,
"is_component_build": true,
"is_debug": false,
"optimize_for_fuzzing": true,
"target_cpu": "x64",
"target_os": "linux",
"use_centipede": true,
"use_reclient": false,
"use_remoteexec": true,
"use_siso": true
}
}

@ -0,0 +1,75 @@
{
"$build/chromium_tests_builder_config": {
"builder_config": {
"additional_exclusions": [
"infra/config/generated/builders/ci/Centipede High End Upload Linux ASan DCheck/gn-args.json"
],
"builder_db": {
"entries": [
{
"builder_id": {
"bucket": "ci",
"builder": "Centipede High End Upload Linux ASan DCheck",
"project": "chromium"
},
"builder_spec": {
"builder_group": "chromium.fuzz",
"execution_mode": "COMPILE_AND_TEST",
"legacy_chromium_config": {
"apply_configs": [
"clobber",
"mb"
],
"build_config": "Release",
"config": "chromium_clang",
"target_bits": 64,
"target_platform": "linux"
},
"legacy_gclient_config": {
"config": "chromium"
}
}
}
]
},
"builder_ids": [
{
"bucket": "ci",
"builder": "Centipede High End Upload Linux ASan DCheck",
"project": "chromium"
}
],
"mirroring_builder_group_and_names": [
{
"builder": "linux-centipede-high-end-asan-dcheck",
"group": "tryserver.chromium.fuzz"
}
],
"retry_failed_shards": true
}
},
"$build/siso": {
"configs": [
"builder"
],
"enable_cloud_monitoring": true,
"enable_cloud_profiler": true,
"enable_cloud_trace": true,
"experiments": [],
"metrics_project": "chromium-reclient-metrics",
"project": "rbe-chromium-trusted",
"remote_jobs": 250
},
"$recipe_engine/resultdb/test_presentation": {
"column_keys": [],
"grouping_keys": [
"status",
"v.test_suite"
]
},
"archive_prefix": "centipede-high-end-dcheck",
"builder_group": "chromium.fuzz",
"recipe": "chromium/fuzz",
"upload_bucket": "chromium-browser-centipede",
"upload_directory": "asan"
}

@ -0,0 +1,14 @@
{
"$build/siso": {
"configs": [
"builder"
],
"enable_cloud_monitoring": true,
"enable_cloud_profiler": true,
"enable_cloud_trace": true,
"experiments": [],
"metrics_project": "chromium-reclient-metrics",
"project": "rbe-chromium-untrusted",
"remote_jobs": 250
}
}

@ -269,6 +269,7 @@
"ASan Release (32-bit x86 with V8-ARM)": "ci/ASan Release (32-bit x86 with V8-ARM)/gn-args.json",
"ASan Release Media (32-bit x86 with V8-ARM)": "ci/ASan Release Media (32-bit x86 with V8-ARM)/gn-args.json",
"Centipede High End Upload Linux ASan": "ci/Centipede High End Upload Linux ASan/gn-args.json",
"Centipede High End Upload Linux ASan DCheck": "ci/Centipede High End Upload Linux ASan DCheck/gn-args.json",
"Centipede Upload Linux ASan": "ci/Centipede Upload Linux ASan/gn-args.json",
"ChromiumOS ASAN Release": "ci/ChromiumOS ASAN Release/gn-args.json",
"Libfuzzer High End Upload Linux ASan": "ci/Libfuzzer High End Upload Linux ASan/gn-args.json",
@ -780,6 +781,7 @@
"linux-asan-v8-arm-dbg": "try/linux-asan-v8-arm-dbg/gn-args.json",
"linux-asan-v8-arm-rel": "try/linux-asan-v8-arm-rel/gn-args.json",
"linux-asan-v8-sandbox-testing": "try/linux-asan-v8-sandbox-testing/gn-args.json",
"linux-centipede-high-end-asan-dcheck": "try/linux-centipede-high-end-asan-dcheck/gn-args.json",
"linux-chromeos-asan-rel": "try/linux-chromeos-asan-rel/gn-args.json",
"linux-libfuzzer-high-end-asan-dbg": "try/linux-libfuzzer-high-end-asan-dbg/gn-args.json",
"linux-libfuzzer-high-end-asan-rel": "try/linux-libfuzzer-high-end-asan-rel/gn-args.json",

@ -0,0 +1,19 @@
{
"gn_args": {
"archive_seed_corpus": false,
"dcheck_always_on": true,
"generate_fuzzer_owners": false,
"high_end_fuzzer_targets": true,
"is_asan": true,
"is_component_build": true,
"is_debug": false,
"optimize_for_fuzzing": true,
"symbol_level": 0,
"target_cpu": "x64",
"target_os": "linux",
"use_centipede": true,
"use_reclient": false,
"use_remoteexec": true,
"use_siso": true
}
}

@ -0,0 +1,65 @@
{
"$build/chromium_tests_builder_config": {
"builder_config": {
"additional_exclusions": [
"infra/config/generated/builders/try/linux-centipede-high-end-asan-dcheck/gn-args.json"
],
"builder_db": {
"entries": [
{
"builder_id": {
"bucket": "ci",
"builder": "Centipede High End Upload Linux ASan DCheck",
"project": "chromium"
},
"builder_spec": {
"builder_group": "chromium.fuzz",
"execution_mode": "COMPILE_AND_TEST",
"legacy_chromium_config": {
"apply_configs": [
"clobber",
"mb"
],
"build_config": "Release",
"config": "chromium_clang",
"target_bits": 64,
"target_platform": "linux"
},
"legacy_gclient_config": {
"config": "chromium"
}
}
}
]
},
"builder_ids": [
{
"bucket": "ci",
"builder": "Centipede High End Upload Linux ASan DCheck",
"project": "chromium"
}
]
}
},
"$build/siso": {
"configs": [
"builder"
],
"enable_cloud_monitoring": true,
"enable_cloud_profiler": true,
"enable_cloud_trace": true,
"experiments": [],
"metrics_project": "chromium-reclient-metrics",
"project": "rbe-chromium-untrusted",
"remote_jobs": -1
},
"$recipe_engine/resultdb/test_presentation": {
"column_keys": [],
"grouping_keys": [
"status",
"v.test_suite"
]
},
"builder_group": "tryserver.chromium.fuzz",
"recipe": "chromium_trybot"
}

@ -726,6 +726,27 @@
}
]
},
"Centipede High End Upload Linux ASan DCheck": {
"contact_team_email": "chrome-deet-core@google.com",
"problem_specs": [
{
"name": "Unhealthy",
"period_days": 7,
"score": 5,
"thresholds": {
"_default": "_default"
}
},
{
"name": "Low Value",
"period_days": 90,
"score": 1,
"thresholds": {
"_default": "_default"
}
}
]
},
"Centipede Upload Linux ASan": {
"contact_team_email": "chrome-deet-core@google.com",
"problem_specs": [

@ -4887,6 +4887,11 @@ config_groups {
includable_only: true
disable_reuse_footers: "Include-Ci-Only-Tests"
}
builders {
name: "chromium/try/linux-centipede-high-end-asan-dcheck"
includable_only: true
disable_reuse_footers: "Include-Ci-Only-Tests"
}
builders {
name: "chromium/try/linux-cfm-rel"
disable_reuse_footers: "Include-Ci-Only-Tests"

@ -5339,6 +5339,117 @@ buckets {
predicates: "string(build.output.properties.is_cached) == \"false\""
}
}
builders {
name: "Centipede High End Upload Linux ASan DCheck"
swarming_host: "chromium-swarm.appspot.com"
dimensions: "builderless:1"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
dimensions: "free_space:standard"
dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.chromium.ci"
dimensions: "ssd:0"
exe {
cipd_package: "infra/chromium/bootstrapper/${platform}"
cipd_version: "latest"
cmd: "bootstrapper"
}
properties:
'{'
' "$bootstrap/exe": {'
' "exe": {'
' "cipd_package": "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",'
' "cipd_version": "refs/heads/main",'
' "cmd": ['
' "luciexe"'
' ]'
' }'
' },'
' "$bootstrap/properties": {'
' "properties_file": "infra/config/generated/builders/ci/Centipede High End Upload Linux ASan DCheck/properties.json",'
' "shadow_properties_file": "infra/config/generated/builders/ci/Centipede High End Upload Linux ASan DCheck/shadow-properties.json",'
' "top_level_project": {'
' "ref": "refs/heads/main",'
' "repo": {'
' "host": "chromium.googlesource.com",'
' "project": "chromium/src"'
' }'
' }'
' },'
' "builder_group": "chromium.fuzz",'
' "led_builder_is_bootstrapped": true,'
' "recipe": "chromium/fuzz"'
'}'
execution_timeout_secs: 10800
build_numbers: YES
service_account: "chromium-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
experiments {
key: "chromium.use_per_builder_build_dir_name"
value: 100
}
experiments {
key: "luci.recipes.use_python3"
value: 100
}
resultdb {
enable: true
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "ci_test_results"
test_results {}
}
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "gpu_ci_test_results"
test_results {
predicate {
test_id_regexp: "ninja://(chrome|content)/test:telemetry_gpu_integration_test[^/]*/.+"
}
}
}
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "blink_web_tests_ci_test_results"
test_results {
predicate {
test_id_regexp: "(ninja://[^/]*blink_web_tests/.+)|(ninja://[^/]*_wpt_tests/.+)|(ninja://[^/]*headless_shell_wpt/.+)"
}
}
}
history_options {
use_invocation_timestamp: true
}
}
description_html: "This builder uploads centipede high end fuzzers in release mode with dcheck_always_on.<br/>This builder is mirrored by any of the following try builders:<br/><ul><li><a href=\"https://ci.chromium.org/p/chromium/builders/try/linux-centipede-high-end-asan-dcheck\">linux-centipede-high-end-asan-dcheck</a></li></ul><br/>Builder owner: <a href=mailto:chrome-deet-core@google.com>chrome-deet-core@google.com</a>"
shadow_builder_adjustments {
service_account: "chromium-try-builder@chops-service-accounts.iam.gserviceaccount.com"
pool: "luci.chromium.try"
dimensions: "free_space:"
dimensions: "pool:luci.chromium.try"
}
contact_team_email: "chrome-deet-core@google.com"
custom_metric_definitions {
name: "/chrome/infra/browser/builds/cached_count"
predicates: "has(build.output.properties.is_cached)"
predicates: "string(build.output.properties.is_cached) == \"true\""
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/ran_tests_retry_shard_count"
predicates: "has(build.output.properties.ran_tests_retry_shard)"
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/ran_tests_without_patch_count"
predicates: "has(build.output.properties.ran_tests_without_patch)"
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/uncached_count"
predicates: "has(build.output.properties.is_cached)"
predicates: "string(build.output.properties.is_cached) == \"false\""
}
}
builders {
name: "Centipede Upload Linux ASan"
swarming_host: "chromium-swarm.appspot.com"
@ -110895,6 +111006,118 @@ buckets {
predicates: "string(build.output.properties.is_cached) == \"false\""
}
}
builders {
name: "linux-centipede-high-end-asan-dcheck"
swarming_host: "chromium-swarm.appspot.com"
dimensions: "builderless:1"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
dimensions: "free_space:standard"
dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.chromium.try"
dimensions: "ssd:0"
exe {
cipd_package: "infra/chromium/bootstrapper/${platform}"
cipd_version: "latest"
cmd: "bootstrapper"
}
properties:
'{'
' "$bootstrap/exe": {'
' "exe": {'
' "cipd_package": "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",'
' "cipd_version": "refs/heads/main",'
' "cmd": ['
' "luciexe"'
' ]'
' }'
' },'
' "$bootstrap/properties": {'
' "properties_file": "infra/config/generated/builders/try/linux-centipede-high-end-asan-dcheck/properties.json",'
' "top_level_project": {'
' "ref": "refs/heads/main",'
' "repo": {'
' "host": "chromium.googlesource.com",'
' "project": "chromium/src"'
' }'
' }'
' },'
' "builder_group": "tryserver.chromium.fuzz",'
' "led_builder_is_bootstrapped": true,'
' "recipe": "chromium_trybot"'
'}'
execution_timeout_secs: 14400
expiration_secs: 7200
grace_period {
seconds: 120
}
build_numbers: YES
service_account: "chromium-try-builder@chops-service-accounts.iam.gserviceaccount.com"
experiments {
key: "chromium.use_per_builder_build_dir_name"
value: 100
}
experiments {
key: "luci.buildbucket.canary_software"
value: 5
}
experiments {
key: "luci.recipes.use_python3"
value: 100
}
resultdb {
enable: true
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "try_test_results"
test_results {}
}
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "gpu_try_test_results"
test_results {
predicate {
test_id_regexp: "ninja://(chrome|content)/test:telemetry_gpu_integration_test[^/]*/.+"
}
}
}
bq_exports {
project: "chrome-luci-data"
dataset: "chromium"
table: "blink_web_tests_try_test_results"
test_results {
predicate {
test_id_regexp: "(ninja://[^/]*blink_web_tests/.+)|(ninja://[^/]*_wpt_tests/.+)|(ninja://[^/]*headless_shell_wpt/.+)"
}
}
}
history_options {
use_invocation_timestamp: true
}
}
description_html: "<br>This builder uploads centipede high end fuzzers in release mode with dcheck_always_on.<br/><br/>This builder mirrors the following CI builders:<br/><ul><li><a href=\"https://ci.chromium.org/p/chromium/builders/ci/Centipede High End Upload Linux ASan DCheck\">Centipede High End Upload Linux ASan DCheck</a></li></ul><br/>Builder owner: <a href=mailto:chrome-deet-core@google.com>chrome-deet-core@google.com</a>"
contact_team_email: "chrome-deet-core@google.com"
custom_metric_definitions {
name: "/chrome/infra/browser/builds/cached_count"
predicates: "has(build.output.properties.is_cached)"
predicates: "string(build.output.properties.is_cached) == \"true\""
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/ran_tests_retry_shard_count"
predicates: "has(build.output.properties.ran_tests_retry_shard)"
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/ran_tests_without_patch_count"
predicates: "has(build.output.properties.ran_tests_without_patch)"
}
custom_metric_definitions {
name: "/chrome/infra/browser/builds/uncached_count"
predicates: "has(build.output.properties.is_cached)"
predicates: "string(build.output.properties.is_cached) == \"false\""
}
}
builders {
name: "linux-cfm-rel"
swarming_host: "chromium-swarm.appspot.com"

@ -6638,12 +6638,12 @@ consoles {
builders {
name: "buildbucket/luci.chromium.ci/Centipede Upload Linux ASan"
category: "chromium.fuzz|centipede"
short_name: "centipede"
short_name: "cent"
}
builders {
name: "buildbucket/luci.chromium.ci/Centipede High End Upload Linux ASan"
category: "chromium.fuzz|centipede"
short_name: "centipede high end"
short_name: "cent high"
}
builders {
name: "buildbucket/luci.chromium.ci/ChromiumOS ASAN Release"
@ -14597,12 +14597,17 @@ consoles {
builders {
name: "buildbucket/luci.chromium.ci/Centipede Upload Linux ASan"
category: "centipede"
short_name: "centipede"
short_name: "cent"
}
builders {
name: "buildbucket/luci.chromium.ci/Centipede High End Upload Linux ASan"
category: "centipede"
short_name: "centipede high end"
short_name: "cent high"
}
builders {
name: "buildbucket/luci.chromium.ci/Centipede High End Upload Linux ASan DCheck"
category: "centipede"
short_name: "cent high dc"
}
builders {
name: "buildbucket/luci.chromium.ci/Win ASan Release"
@ -27294,6 +27299,9 @@ consoles {
builders {
name: "buildbucket/luci.chromium.try/linux-centipede-asan-rel"
}
builders {
name: "buildbucket/luci.chromium.try/linux-centipede-high-end-asan-dcheck"
}
builders {
name: "buildbucket/luci.chromium.try/linux-cfm-rel"
}
@ -28718,6 +28726,9 @@ consoles {
builders {
name: "buildbucket/luci.chromium.try/linux-asan-v8-sandbox-testing"
}
builders {
name: "buildbucket/luci.chromium.try/linux-centipede-high-end-asan-dcheck"
}
builders {
name: "buildbucket/luci.chromium.try/linux-chromeos-asan-rel"
}

@ -293,6 +293,19 @@ notifiers {
repository: "https://chromium.googlesource.com/chromium/src"
}
}
notifiers {
notifications {
on_change: true
email {
recipients: "chrome-fuzzing-core+bots@google.com"
}
}
builders {
bucket: "ci"
name: "Centipede High End Upload Linux ASan DCheck"
repository: "https://chromium.googlesource.com/chromium/src"
}
}
notifiers {
notifications {
on_change: true

@ -333,6 +333,18 @@ job {
builder: "Centipede High End Upload Linux ASan"
}
}
job {
id: "Centipede High End Upload Linux ASan DCheck"
realm: "ci"
triggering_policy {
kind: GREEDY_BATCHING
}
buildbucket {
server: "cr-buildbucket.appspot.com"
bucket: "ci"
builder: "Centipede High End Upload Linux ASan DCheck"
}
}
job {
id: "Centipede Upload Linux ASan"
realm: "ci"
@ -6650,6 +6662,7 @@ trigger {
triggers: "CFI Linux CF"
triggers: "CFI Linux ToT"
triggers: "Centipede High End Upload Linux ASan"
triggers: "Centipede High End Upload Linux ASan DCheck"
triggers: "Centipede Upload Linux ASan"
triggers: "ChromeOS FYI Release (amd64-generic)"
triggers: "ChromeOS FYI Release Skylab (volteer)"

@ -1198,6 +1198,14 @@ gn_args.config(
],
)
gn_args.config(
name = "release_with_dchecks",
args = {
"is_debug": False,
"dcheck_always_on": True,
},
)
gn_args.config(
name = "resource_allowlisting",
args = {

@ -392,7 +392,7 @@ ci.builder(
),
console_view_entry = consoles.console_view_entry(
category = "centipede",
short_name = "centipede",
short_name = "cent",
),
contact_team_email = "chrome-deet-core@google.com",
execution_timeout = 4 * time.hour,
@ -446,7 +446,7 @@ Those fuzzers require more resources to run correctly.\
),
console_view_entry = consoles.console_view_entry(
category = "centipede",
short_name = "centipede high end",
short_name = "cent high",
),
contact_team_email = "chrome-deet-core@google.com",
properties = {
@ -456,6 +456,56 @@ Those fuzzers require more resources to run correctly.\
},
)
ci.builder(
name = "Centipede High End Upload Linux ASan DCheck",
description_html = """This builder uploads centipede high end fuzzers \
in release mode with dcheck_always_on.\
""",
executable = "recipe:chromium/fuzz",
triggering_policy = scheduler.greedy_batching(),
builder_spec = builder_config.builder_spec(
gclient_config = builder_config.gclient_config(
config = "chromium",
),
chromium_config = builder_config.chromium_config(
config = "chromium_clang",
apply_configs = [
"clobber",
"mb",
],
build_config = builder_config.build_config.RELEASE,
target_bits = 64,
target_platform = builder_config.target_platform.LINUX,
),
),
gn_args = gn_args.config(
configs = [
"asan",
"centipede",
"disable_seed_corpus",
"high_end_fuzzer_targets",
"linux",
"optimize_for_fuzzing",
"release_with_dchecks",
"remoteexec",
"shared",
"x64",
],
),
# TODO(crbug.com/399002817): add this to the gardener_rotations.
gardener_rotations = args.ignore_default(None),
console_view_entry = consoles.console_view_entry(
category = "centipede",
short_name = "cent high dc",
),
contact_team_email = "chrome-deet-core@google.com",
properties = {
"upload_bucket": "chromium-browser-centipede",
"upload_directory": "asan",
"archive_prefix": "centipede-high-end-dcheck",
},
)
ci.builder(
name = "Libfuzzer High End Upload Linux ASan",
description_html = """This builder uploads centipede high end fuzzers.\

@ -143,6 +143,19 @@ try_.builder(
os = os.WINDOWS_DEFAULT,
)
try_.builder(
name = "linux-centipede-high-end-asan-dcheck",
mirrors = ["ci/Centipede High End Upload Linux ASan DCheck"],
gn_args = gn_args.config(
configs = [
"ci/Centipede High End Upload Linux ASan DCheck",
"no_symbols",
"skip_generate_fuzzer_owners",
],
),
contact_team_email = "chrome-deet-core@google.com",
)
try_.builder(
name = "linux-libfuzzer-high-end-asan-rel",
mirrors = ["ci/Libfuzzer High End Upload Linux ASan"],

@ -47,11 +47,14 @@ import("//testing/test.gni")
template("fuzzer_test") {
_high_end_job_required = false
if (defined(invoker.high_end_job_required) && invoker.high_end_job_required) {
_high_end_job_required = !is_debug
# Target should run in pure release mode, i.e. no debug and no dchecks.
_high_end_job_required = !is_debug && !dcheck_always_on
}
if (defined(invoker.high_end_debug_job_required) &&
invoker.high_end_debug_job_required) {
_high_end_job_required = _high_end_job_required || is_debug
# Target should run in debug'ish mode, i.e. with debug or dchecks.
_high_end_job_required =
_high_end_job_required || is_debug || dcheck_always_on
}
# If the job is a high_end job and that we are currently building a high_end