From 8515e3b7c2837562395acd236c504a59328d924d Mon Sep 17 00:00:00 2001 From: Elliott Brooks <21270878+elliette@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:53:48 -0700 Subject: [PATCH 1/3] Initial implementation of cost-based sharding for web tests --- .ci.yaml | 36 +--- script/configs/dart_unit_tests_web_costs.yaml | 6 + script/tool/CHANGELOG.md | 2 + .../tool/lib/src/common/package_command.dart | 200 ++++++++++++++---- .../src/common/package_looping_command.dart | 10 + 5 files changed, 192 insertions(+), 62 deletions(-) create mode 100644 script/configs/dart_unit_tests_web_costs.yaml diff --git a/.ci.yaml b/.ci.yaml index f24589b43c5c..aa0f5f66d713 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -189,7 +189,7 @@ targets: target_file: web_dart_unit_tests.yaml channel: master version_file: flutter_master.version - package_sharding: "--shardIndex 0 --shardCount 3" + package_sharding: "--shardIndex 0 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml" - name: Linux_web web_dart_unit_test_shard_2 master recipe: packages/packages @@ -198,7 +198,7 @@ targets: target_file: web_dart_unit_tests.yaml channel: master version_file: flutter_master.version - package_sharding: "--shardIndex 1 --shardCount 3" + package_sharding: "--shardIndex 1 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml" - name: Linux_web web_dart_unit_test_shard_3 master recipe: packages/packages @@ -207,7 +207,7 @@ targets: target_file: web_dart_unit_tests.yaml channel: master version_file: flutter_master.version - package_sharding: "--shardIndex 2 --shardCount 3" + package_sharding: "--shardIndex 2 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml" - name: Linux_web web_dart_unit_test_shard_1 stable recipe: packages/packages @@ -216,12 +216,12 @@ targets: target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version - package_sharding: "--shardIndex 0 --shardCount 3" + package_sharding: "--shardIndex 0 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and # flutter/flutter#190584) which pass --disable-search-engine-choice-screen # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 - dependencies: >- + dependencies: &chrome_and_driver_stable >- [ {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} ] @@ -233,15 +233,8 @@ targets: target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version - package_sharding: "--shardIndex 1 --shardCount 3" - # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the - # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and - # flutter/flutter#190584) which pass --disable-search-engine-choice-screen - # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 - dependencies: >- - [ - {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} - ] + package_sharding: "--shardIndex 1 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" + dependencies: *chrome_and_driver_stable - name: Linux_web web_dart_unit_test_shard_3 stable recipe: packages/packages @@ -250,15 +243,8 @@ targets: target_file: web_dart_unit_tests.yaml channel: stable version_file: flutter_stable.version - package_sharding: "--shardIndex 2 --shardCount 3" - # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the - # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and - # flutter/flutter#190584) which pass --disable-search-engine-choice-screen - # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 - dependencies: >- - [ - {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} - ] + package_sharding: "--shardIndex 2 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" + dependencies: *chrome_and_driver_stable # Wasm unit tests in master # TODO(stuartmorgan): Switch to testing wasm on master and stable and non-wasm @@ -272,7 +258,7 @@ targets: target_file: web_dart_unit_tests_wasm.yaml channel: master version_file: flutter_master.version - package_sharding: "--shardIndex 0 --shardCount 2" + package_sharding: "--shardIndex 0 --shardCount 2 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" - name: Linux_web web_dart_unit_test_wasm_shard_2 master recipe: packages/packages @@ -281,7 +267,7 @@ targets: target_file: web_dart_unit_tests_wasm.yaml channel: master version_file: flutter_master.version - package_sharding: "--shardIndex 1 --shardCount 2" + package_sharding: "--shardIndex 1 --shardCount 2 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" - name: Linux analyze master recipe: packages/packages diff --git a/script/configs/dart_unit_tests_web_costs.yaml b/script/configs/dart_unit_tests_web_costs.yaml new file mode 100644 index 000000000000..8eb22c43293f --- /dev/null +++ b/script/configs/dart_unit_tests_web_costs.yaml @@ -0,0 +1,6 @@ +# Based on cost analysis in https://github.com/flutter/flutter/issues/189278. +packages/material_ui: 52 +packages/material_ui/example: 32 +# Based on the relative size of cupertino_ui vs material_ui. +packages/cupertino_ui: 18 +packages/cupertino_ui/example: 12 diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md index 82ddb260f8fe..07de129d5ef5 100644 --- a/script/tool/CHANGELOG.md +++ b/script/tool/CHANGELOG.md @@ -1,6 +1,8 @@ ## NEXT * Adds support for batch release of pre-1.0 packages. +* Adds `--shardCost` argument to support cost-weighted greedy sharding across + packages and subpackages. ## 0.14.4 diff --git a/script/tool/lib/src/common/package_command.dart b/script/tool/lib/src/common/package_command.dart index 76546964b962..2aaf1b0f4c54 100644 --- a/script/tool/lib/src/common/package_command.dart +++ b/script/tool/lib/src/common/package_command.dart @@ -10,6 +10,7 @@ import 'package:file/file.dart'; import 'package:git/git.dart'; import 'package:path/path.dart' as p; import 'package:platform/platform.dart'; +import 'package:yaml/yaml.dart'; import 'core.dart'; import 'file_utils.dart'; @@ -70,6 +71,15 @@ abstract class PackageCommand extends Command { valueHelp: 'n', defaultsTo: '1', ); + argParser.addOption( + _shardCostArg, + help: + 'A path to a YAML file mapping repository-relative package/subpackage ' + 'paths (e.g., "packages/foo", "packages/foo/example") to relative cost units for weighted sharding.\n\n' + 'Unlisted packages default to a cost of 1.0. When provided, packages ' + 'and subpackages are sharded using cost-weighted greedy bin-packing.', + valueHelp: 'path/to/cost_map.yaml', + ); argParser.addFlag( _exactMatchOnlyArg, help: 'Disables package group matching in package selection.', @@ -186,6 +196,7 @@ abstract class PackageCommand extends Command { // Sharding. static const String _shardCountArg = 'shardCount'; static const String _shardIndexArg = 'shardIndex'; + static const String _shardCostArg = 'shardCost'; // Utility. static const String _logTimingArg = 'log-timing'; @@ -336,7 +347,44 @@ abstract class PackageCommand extends Command { return excludedPackages; } - /// Returns the root diretories of the packages involved in this command + /// Map of relative path to package to unit cost, as determined by the [shardCostArg]. + Map? _shardingCosts; + + /// Returns the sharding cost map, if provided by the [shardCostArg]. + Map? get shardingCosts { + // If we've already calculated the costs, then immediately return them. + if (_shardingCosts != null) { + return _shardingCosts; + } + + // Otherwise parse the shard cost file to calculate the costs. + final String? costFilePath = getNullableStringArg(_shardCostArg); + if (costFilePath == null) { + return null; + } + final File costFile = rootDir.fileSystem.file(costFilePath); + if (!costFile.existsSync()) { + printError('Cost file "$costFilePath" does not exist.'); + throw ToolExit(exitInvalidArguments); + } + final Object? yamlContent = loadYaml(costFile.readAsStringSync()); + if (yamlContent is! YamlMap) { + printError('Cost file "$costFilePath" must be a YAML map of package names to numeric costs.'); + throw ToolExit(exitInvalidArguments); + } + final costs = {}; + for (final MapEntry entry in yamlContent.entries) { + final packagePath = entry.key.toString(); + final packageCost = entry.value as num?; + if (packageCost != null) { + costs[packagePath] = packageCost.toDouble(); + } + } + _shardingCosts = costs; + return _shardingCosts; + } + + /// Returns the root directories of the packages involved in this command /// execution. /// /// Depending on the command arguments, this may be a user-specified set of @@ -345,27 +393,8 @@ abstract class PackageCommand extends Command { /// /// By default, packages excluded via --exclude will not be in the stream, but /// they can be included by passing false for [filterExcluded]. - Stream getTargetPackages({bool filterExcluded = true}) async* { - // To avoid assuming consistency of `Directory.list` across command - // invocations, we collect and sort the package folders before sharding. - // This is considered an implementation detail which is why the API still - // uses streams. - final List allPackages = await _getAllPackages().toList(); - allPackages.sort( - (PackageEnumerationEntry p1, PackageEnumerationEntry p2) => - p1.package.path.compareTo(p2.package.path), - ); - final int shardSize = - allPackages.length ~/ shardCount + (allPackages.length % shardCount == 0 ? 0 : 1); - final int start = min(shardIndex * shardSize, allPackages.length); - final int end = min(start + shardSize, allPackages.length); - - for (final PackageEnumerationEntry package in allPackages.sublist(start, end)) { - if (!(filterExcluded && package.excluded)) { - yield package; - } - } - } + Stream getTargetPackages({bool filterExcluded = true}) => + _getTargetPackages(filterExcluded: filterExcluded); /// Returns the root Dart package folders of the packages involved in this /// command execution, assuming there is only one shard. Depending on the @@ -597,21 +626,8 @@ abstract class PackageCommand extends Command { /// /// Subpackages are guaranteed to be after the containing package in the /// stream. - Stream getTargetPackagesAndSubpackages({ - bool filterExcluded = true, - }) async* { - await for (final PackageEnumerationEntry package in getTargetPackages( - filterExcluded: filterExcluded, - )) { - yield package; - yield* Stream.fromIterable( - package.package.getSubpackages().map( - (RepositoryPackage subPackage) => - PackageEnumerationEntry(subPackage, excluded: package.excluded), - ), - ); - } - } + Stream getTargetPackagesAndSubpackages({bool filterExcluded = true}) => + _getTargetPackages(includeSubpackages: true, filterExcluded: filterExcluded); /// Returns the files contained, recursively, within the packages /// involved in this command execution. @@ -644,6 +660,116 @@ abstract class PackageCommand extends Command { return gitVersionFinder; } + /// Helper to return target packages for this shard. + Stream _getTargetPackages({ + bool includeSubpackages = false, + bool filterExcluded = true, + }) async* { + final hasCosts = shardingCosts != null; + final List packagesToShard = + await (hasCosts ? _getAllPackagesAndSubpackages() : _getAllPackages()).toList(); + packagesToShard.sort( + (PackageEnumerationEntry p1, PackageEnumerationEntry p2) => + p1.package.path.compareTo(p2.package.path), + ); + final List sharded = _shardPackageEntries(packagesToShard); + + for (final entry in sharded) { + if (filterExcluded && entry.excluded) { + continue; + } + + if (includeSubpackages || entry.package.isTopLevel) { + yield entry; + if (!hasCosts && includeSubpackages) { + yield* _getSubpackages(entry); + } + } + } + } + + double _getPackageCost(RepositoryPackage package) { + final String relPath = relativePosixPath( + package.directory, + from: rootDir, + platformContext: path, + ); + return shardingCosts?[relPath] ?? 1.0; + } + + List _shardPackageEntries(List entries) { + if (entries.isEmpty || shardCount == 1) { + return shardIndex == 0 ? entries : []; + } + + return shardingCosts != null + ? _shardByCostAndAlphabetically(entries) + : _shardAlphabetically(entries); + } + + List _shardAlphabetically(List entries) { + final int shardSize = entries.length ~/ shardCount + (entries.length % shardCount == 0 ? 0 : 1); + final int start = min(shardIndex * shardSize, entries.length); + final int end = min(start + shardSize, entries.length); + return entries.sublist(start, end); + } + + // Shard packages by cost then alphabetically using a greedy bin-packing algorithm. + List _shardByCostAndAlphabetically( + List entries, + ) { + // Sort by cost descending, and then alphabetically by package. + final List<({PackageEnumerationEntry entry, double cost})> weighted = entries.map((entry) { + return (entry: entry, cost: _getPackageCost(entry.package)); + }).toList(); + weighted.sort((a, b) { + final int costComparison = b.cost.compareTo(a.cost); + return costComparison != 0 + ? costComparison + : a.entry.package.path.compareTo(b.entry.package.path); + }); + + final shardCosts = List.filled(shardCount, 0.0); + final shardBuckets = List>.generate( + shardCount, + (_) => [], + ); + + // Assign each package to the shard with the current lowest cost. + for (final package in weighted) { + var lowestCostShardIdx = 0; + double lowestCost = shardCosts[0]; + for (var i = 1; i < shardCount; ++i) { + if (shardCosts[i] < lowestCost) { + lowestCost = shardCosts[i]; + lowestCostShardIdx = i; + } + } + shardBuckets[lowestCostShardIdx].add(package.entry); + shardCosts[lowestCostShardIdx] += package.cost; + } + + final List shard = shardBuckets[shardIndex]; + shard.sort((a, b) => a.package.path.compareTo(b.package.path)); + return shard; + } + + Stream _getAllPackagesAndSubpackages() async* { + await for (final PackageEnumerationEntry entry in _getAllPackages()) { + yield entry; + yield* _getSubpackages(entry); + } + } + + /// Yields the subpackages for a given entry. + Stream _getSubpackages(PackageEnumerationEntry entry) async* { + yield* Stream.fromIterable( + entry.package.getSubpackages().map( + (subPackage) => PackageEnumerationEntry(subPackage, excluded: entry.excluded), + ), + ); + } + // Returns the names of packages that have been changed given a list of // changed files. // diff --git a/script/tool/lib/src/common/package_looping_command.dart b/script/tool/lib/src/common/package_looping_command.dart index 0d7f528da23f..d3d409b15fa5 100644 --- a/script/tool/lib/src/common/package_looping_command.dart +++ b/script/tool/lib/src/common/package_looping_command.dart @@ -139,6 +139,16 @@ abstract class PackageLoopingCommand extends PackageCommand { case PackageLoopingType.topLevelOnly: yield* getTargetPackages(filterExcluded: false); case PackageLoopingType.includeExamples: + // TODO(elliottbrooks): Support includeExamples with sharding costs if a + // command ever needs it. Currently, weighted sharding distributes all + // packages and subpackages across shards individually, so filtering to + // only direct examples would require extra logic here. + if (shardingCosts != null) { + throw UnsupportedError( + 'PackageLoopingType.includeExamples is not supported with custom sharding costs. ' + 'Use PackageLoopingType.includeAllSubpackages instead.', + ); + } await for (final PackageEnumerationEntry packageEntry in getTargetPackages( filterExcluded: false, )) { From f58e20f46026eb548bb877ed2ba99e7eae9f7c22 Mon Sep 17 00:00:00 2001 From: Elliott Brooks <21270878+elliette@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:58:00 -0700 Subject: [PATCH 2/3] Revert accidental unrelated changes --- .ci.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index aa0f5f66d713..1accd293d5f7 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -234,7 +234,14 @@ targets: channel: stable version_file: flutter_stable.version package_sharding: "--shardIndex 1 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" - dependencies: *chrome_and_driver_stable + # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the + # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and + # flutter/flutter#190584) which pass --disable-search-engine-choice-screen + # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 + dependencies: >- + [ + {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} + ] - name: Linux_web web_dart_unit_test_shard_3 stable recipe: packages/packages @@ -244,7 +251,14 @@ targets: channel: stable version_file: flutter_stable.version package_sharding: "--shardIndex 2 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml" - dependencies: *chrome_and_driver_stable + # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the + # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and + # flutter/flutter#190584) which pass --disable-search-engine-choice-screen + # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 + dependencies: >- + [ + {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} + ] # Wasm unit tests in master # TODO(stuartmorgan): Switch to testing wasm on master and stable and non-wasm From b28e4812eb52a2833a57c7615c7e93926ca67e18 Mon Sep 17 00:00:00 2001 From: Elliott Brooks <21270878+elliette@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:00:05 -0700 Subject: [PATCH 3/3] One more accidental edit --- .ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci.yaml b/.ci.yaml index 1accd293d5f7..80a4b0802ee9 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -221,7 +221,7 @@ targets: # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and # flutter/flutter#190584) which pass --disable-search-engine-choice-screen # and --disable-background-timer-throttling; see https://github.com/flutter/flutter/issues/191299 - dependencies: &chrome_and_driver_stable >- + dependencies: >- [ {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"} ]