[Draft][material_ui] Testing new cost-based sharding of web tests

.ci.yaml

@@ -189,7 +189,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: master
       version_file: flutter_master.version
-      package_sharding: "--shardIndex 0 --shardCount 3"
+      package_sharding: "--shardIndex 0 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml"
 
   - name: Linux_web web_dart_unit_test_shard_2 master
     recipe: packages/packages
@@ -198,7 +198,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: master
       version_file: flutter_master.version
-      package_sharding: "--shardIndex 1 --shardCount 3"
+      package_sharding: "--shardIndex 1 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml"
 
   - name: Linux_web web_dart_unit_test_shard_3 master
     recipe: packages/packages
@@ -207,7 +207,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: master
       version_file: flutter_master.version
-      package_sharding: "--shardIndex 2 --shardCount 3"
+      package_sharding: "--shardIndex 2 --shardCount 3 --shardCost=script/configs/dart_unit_tests_web_costs.yaml"
 
   - name: Linux_web web_dart_unit_test_shard_1 stable
     recipe: packages/packages
@@ -216,7 +216,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: stable
       version_file: flutter_stable.version
-      package_sharding: "--shardIndex 0 --shardCount 3"
+      package_sharding: "--shardIndex 0 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml"
       # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the
       # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and
       # flutter/flutter#190584) which pass --disable-search-engine-choice-screen
@@ -233,7 +233,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: stable
       version_file: flutter_stable.version
-      package_sharding: "--shardIndex 1 --shardCount 3"
+      package_sharding: "--shardIndex 1 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml"
       # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the
       # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and
       # flutter/flutter#190584) which pass --disable-search-engine-choice-screen
@@ -250,7 +250,7 @@ targets:
       target_file: web_dart_unit_tests.yaml
       channel: stable
       version_file: flutter_stable.version
-      package_sharding: "--shardIndex 2 --shardCount 3"
+      package_sharding: "--shardIndex 2 --shardCount 3 --shardCost=script/configs/dart_unit_web_tests_costs.yaml"
       # TODO(mdebbar): Upgrade to Chrome 145 once Flutter stable includes the
       # upstream flutter_tools fixes for Chrome 145 (flutter/flutter#190155 and
       # flutter/flutter#190584) which pass --disable-search-engine-choice-screen
@@ -272,7 +272,7 @@ targets:
       target_file: web_dart_unit_tests_wasm.yaml
       channel: master
       version_file: flutter_master.version
-      package_sharding: "--shardIndex 0 --shardCount 2"
+      package_sharding: "--shardIndex 0 --shardCount 2 --shardCost=script/configs/dart_unit_web_tests_costs.yaml"
 
   - name: Linux_web web_dart_unit_test_wasm_shard_2 master
     recipe: packages/packages
@@ -281,7 +281,7 @@ targets:
       target_file: web_dart_unit_tests_wasm.yaml
       channel: master
       version_file: flutter_master.version
-      package_sharding: "--shardIndex 1 --shardCount 2"
+      package_sharding: "--shardIndex 1 --shardCount 2 --shardCost=script/configs/dart_unit_web_tests_costs.yaml"
 
   - name: Linux analyze master
     recipe: packages/packages

script/configs/dart_unit_tests_web_costs.yaml

@@ -0,0 +1,6 @@
+# Based on cost analysis in https://github.com/flutter/flutter/issues/189278.
+packages/material_ui: 52
+packages/material_ui/example: 32
+# Based on the relative size of cupertino_ui vs material_ui.
+packages/cupertino_ui: 18
+packages/cupertino_ui/example: 12

script/tool/CHANGELOG.md

@@ -1,6 +1,8 @@
 ## NEXT
 
 * Adds support for batch release of pre-1.0 packages.
+* Adds `--shardCost` argument to support cost-weighted greedy sharding across
+  packages and subpackages.
 
 ## 0.14.4
 

script/tool/lib/src/common/package_command.dart

@@ -10,6 +10,7 @@ import 'package:file/file.dart';
 import 'package:git/git.dart';
 import 'package:path/path.dart' as p;
 import 'package:platform/platform.dart';
+import 'package:yaml/yaml.dart';
 
 import 'core.dart';
 import 'file_utils.dart';
@@ -70,6 +71,15 @@ abstract class PackageCommand extends Command<void> {
       valueHelp: 'n',
       defaultsTo: '1',
     );
+    argParser.addOption(
+      _shardCostArg,
+      help:
+          'A path to a YAML file mapping repository-relative package/subpackage '
+          'paths (e.g., "packages/foo", "packages/foo/example") to relative cost units for weighted sharding.\n\n'
+          'Unlisted packages default to a cost of 1.0. When provided, packages '
+          'and subpackages are sharded using cost-weighted greedy bin-packing.',
+      valueHelp: 'path/to/cost_map.yaml',
+    );
     argParser.addFlag(
       _exactMatchOnlyArg,
       help: 'Disables package group matching in package selection.',
@@ -186,6 +196,7 @@ abstract class PackageCommand extends Command<void> {
   // Sharding.
   static const String _shardCountArg = 'shardCount';
   static const String _shardIndexArg = 'shardIndex';
+  static const String _shardCostArg = 'shardCost';
   // Utility.
   static const String _logTimingArg = 'log-timing';
 
@@ -336,7 +347,44 @@ abstract class PackageCommand extends Command<void> {
     return excludedPackages;
   }
 
-  /// Returns the root diretories of the packages involved in this command
+  /// Map of relative path to package to unit cost, as determined by the [shardCostArg].
+  Map<String, double>? _shardingCosts;
+
+  /// Returns the sharding cost map, if provided by the [shardCostArg].
+  Map<String, double>? get shardingCosts {
+    // If we've already calculated the costs, then immediately return them.
+    if (_shardingCosts != null) {
+      return _shardingCosts;
+    }
+
+    // Otherwise parse the shard cost file to calculate the costs.
+    final String? costFilePath = getNullableStringArg(_shardCostArg);
+    if (costFilePath == null) {
+      return null;
+    }
+    final File costFile = rootDir.fileSystem.file(costFilePath);
+    if (!costFile.existsSync()) {
+      printError('Cost file "$costFilePath" does not exist.');
+      throw ToolExit(exitInvalidArguments);
+    }
+    final Object? yamlContent = loadYaml(costFile.readAsStringSync());
+    if (yamlContent is! YamlMap) {
+      printError('Cost file "$costFilePath" must be a YAML map of package names to numeric costs.');
+      throw ToolExit(exitInvalidArguments);
+    }
+    final costs = <String, double>{};
+    for (final MapEntry<dynamic, dynamic> entry in yamlContent.entries) {
+      final packagePath = entry.key.toString();
+      final packageCost = entry.value as num?;
+      if (packageCost != null) {
+        costs[packagePath] = packageCost.toDouble();
+      }
+    }
+    _shardingCosts = costs;
+    return _shardingCosts;
+  }
+
+  /// Returns the root directories of the packages involved in this command
   /// execution.
   ///
   /// Depending on the command arguments, this may be a user-specified set of
@@ -345,27 +393,8 @@ abstract class PackageCommand extends Command<void> {
   ///
   /// By default, packages excluded via --exclude will not be in the stream, but
   /// they can be included by passing false for [filterExcluded].
-  Stream<PackageEnumerationEntry> getTargetPackages({bool filterExcluded = true}) async* {
-    // To avoid assuming consistency of `Directory.list` across command
-    // invocations, we collect and sort the package folders before sharding.
-    // This is considered an implementation detail which is why the API still
-    // uses streams.
-    final List<PackageEnumerationEntry> allPackages = await _getAllPackages().toList();
-    allPackages.sort(
-      (PackageEnumerationEntry p1, PackageEnumerationEntry p2) =>
-          p1.package.path.compareTo(p2.package.path),
-    );
-    final int shardSize =
-        allPackages.length ~/ shardCount + (allPackages.length % shardCount == 0 ? 0 : 1);
-    final int start = min(shardIndex * shardSize, allPackages.length);
-    final int end = min(start + shardSize, allPackages.length);
-
-    for (final PackageEnumerationEntry package in allPackages.sublist(start, end)) {
-      if (!(filterExcluded && package.excluded)) {
-        yield package;
-      }
-    }
-  }
+  Stream<PackageEnumerationEntry> getTargetPackages({bool filterExcluded = true}) =>
+      _getTargetPackages(filterExcluded: filterExcluded);
 
   /// Returns the root Dart package folders of the packages involved in this
   /// command execution, assuming there is only one shard. Depending on the
@@ -597,21 +626,8 @@ abstract class PackageCommand extends Command<void> {
   ///
   /// Subpackages are guaranteed to be after the containing package in the
   /// stream.
-  Stream<PackageEnumerationEntry> getTargetPackagesAndSubpackages({
-    bool filterExcluded = true,
-  }) async* {
-    await for (final PackageEnumerationEntry package in getTargetPackages(
-      filterExcluded: filterExcluded,
-    )) {
-      yield package;
-      yield* Stream<PackageEnumerationEntry>.fromIterable(
-        package.package.getSubpackages().map(
-          (RepositoryPackage subPackage) =>
-              PackageEnumerationEntry(subPackage, excluded: package.excluded),
-        ),
-      );
-    }
-  }
+  Stream<PackageEnumerationEntry> getTargetPackagesAndSubpackages({bool filterExcluded = true}) =>
+      _getTargetPackages(includeSubpackages: true, filterExcluded: filterExcluded);
 
   /// Returns the files contained, recursively, within the packages
   /// involved in this command execution.
@@ -644,6 +660,116 @@ abstract class PackageCommand extends Command<void> {
     return gitVersionFinder;
   }
 
+  /// Helper to return target packages for this shard.
+  Stream<PackageEnumerationEntry> _getTargetPackages({
+    bool includeSubpackages = false,
+    bool filterExcluded = true,
+  }) async* {
+    final hasCosts = shardingCosts != null;
+    final List<PackageEnumerationEntry> packagesToShard =
+        await (hasCosts ? _getAllPackagesAndSubpackages() : _getAllPackages()).toList();
+    packagesToShard.sort(
+      (PackageEnumerationEntry p1, PackageEnumerationEntry p2) =>
+          p1.package.path.compareTo(p2.package.path),
+    );
+    final List<PackageEnumerationEntry> sharded = _shardPackageEntries(packagesToShard);
+
+    for (final entry in sharded) {
+      if (filterExcluded && entry.excluded) {
+        continue;
+      }
+
+      if (includeSubpackages || entry.package.isTopLevel) {
+        yield entry;
+        if (!hasCosts && includeSubpackages) {
+          yield* _getSubpackages(entry);
+        }
+      }
+    }
+  }
+
+  double _getPackageCost(RepositoryPackage package) {
+    final String relPath = relativePosixPath(
+      package.directory,
+      from: rootDir,
+      platformContext: path,
+    );
+    return shardingCosts?[relPath] ?? 1.0;
+  }
+
+  List<PackageEnumerationEntry> _shardPackageEntries(List<PackageEnumerationEntry> entries) {
+    if (entries.isEmpty || shardCount == 1) {
+      return shardIndex == 0 ? entries : <PackageEnumerationEntry>[];
+    }
+
+    return shardingCosts != null
+        ? _shardByCostAndAlphabetically(entries)
+        : _shardAlphabetically(entries);
+  }
+
+  List<PackageEnumerationEntry> _shardAlphabetically(List<PackageEnumerationEntry> entries) {
+    final int shardSize = entries.length ~/ shardCount + (entries.length % shardCount == 0 ? 0 : 1);
+    final int start = min(shardIndex * shardSize, entries.length);
+    final int end = min(start + shardSize, entries.length);
+    return entries.sublist(start, end);
+  }
+
+  // Shard packages by cost then alphabetically using a greedy bin-packing algorithm.
+  List<PackageEnumerationEntry> _shardByCostAndAlphabetically(
+    List<PackageEnumerationEntry> entries,
+  ) {
+    // Sort by cost descending, and then alphabetically by package.
+    final List<({PackageEnumerationEntry entry, double cost})> weighted = entries.map((entry) {
+      return (entry: entry, cost: _getPackageCost(entry.package));
+    }).toList();
+    weighted.sort((a, b) {
+      final int costComparison = b.cost.compareTo(a.cost);
+      return costComparison != 0
+          ? costComparison
+          : a.entry.package.path.compareTo(b.entry.package.path);
+    });
+
+    final shardCosts = List<double>.filled(shardCount, 0.0);
+    final shardBuckets = List<List<PackageEnumerationEntry>>.generate(
+      shardCount,
+      (_) => <PackageEnumerationEntry>[],
+    );
+
+    // Assign each package to the shard with the current lowest cost.
+    for (final package in weighted) {
+      var lowestCostShardIdx = 0;
+      double lowestCost = shardCosts[0];
+      for (var i = 1; i < shardCount; ++i) {
+        if (shardCosts[i] < lowestCost) {
+          lowestCost = shardCosts[i];
+          lowestCostShardIdx = i;
+        }
+      }
+      shardBuckets[lowestCostShardIdx].add(package.entry);
+      shardCosts[lowestCostShardIdx] += package.cost;
+    }
+
+    final List<PackageEnumerationEntry> shard = shardBuckets[shardIndex];
+    shard.sort((a, b) => a.package.path.compareTo(b.package.path));
+    return shard;
+  }
+
+  Stream<PackageEnumerationEntry> _getAllPackagesAndSubpackages() async* {
+    await for (final PackageEnumerationEntry entry in _getAllPackages()) {
+      yield entry;
+      yield* _getSubpackages(entry);
+    }
+  }
+
+  /// Yields the subpackages for a given entry.
+  Stream<PackageEnumerationEntry> _getSubpackages(PackageEnumerationEntry entry) async* {
+    yield* Stream<PackageEnumerationEntry>.fromIterable(
+      entry.package.getSubpackages().map(
+        (subPackage) => PackageEnumerationEntry(subPackage, excluded: entry.excluded),
+      ),
+    );
+  }
+
   // Returns the names of packages that have been changed given a list of
   // changed files.
   //

script/tool/lib/src/common/package_looping_command.dart

@@ -139,6 +139,16 @@ abstract class PackageLoopingCommand extends PackageCommand {
       case PackageLoopingType.topLevelOnly:
         yield* getTargetPackages(filterExcluded: false);
       case PackageLoopingType.includeExamples:
+        // TODO(elliottbrooks): Support includeExamples with sharding costs if a
+        // command ever needs it. Currently, weighted sharding distributes all
+        // packages and subpackages across shards individually, so filtering to
+        // only direct examples would require extra logic here.
+        if (shardingCosts != null) {
+          throw UnsupportedError(
+            'PackageLoopingType.includeExamples is not supported with custom sharding costs. '
+            'Use PackageLoopingType.includeAllSubpackages instead.',
+          );
+        }
         await for (final PackageEnumerationEntry packageEntry in getTargetPackages(
           filterExcluded: false,
         )) {