diff --git a/conf/config.neon b/conf/config.neon
index de093affa36..31869418817 100644
--- a/conf/config.neon
+++ b/conf/config.neon
@@ -107,7 +107,7 @@ parameters:
parallel:
jobSize: 20
processTimeout: 600.0
- maximumNumberOfProcesses: 8
+ maximumNumberOfProcesses: auto
minimumNumberOfJobsPerProcess: 2
buffer: 134217728 # 128 MB
loadLimit: 1.0
diff --git a/conf/parametersSchema.neon b/conf/parametersSchema.neon
index ecbac473b43..5741d05cd34 100644
--- a/conf/parametersSchema.neon
+++ b/conf/parametersSchema.neon
@@ -109,7 +109,7 @@ parametersSchema:
parallel: structure([
jobSize: int(),
processTimeout: float(),
- maximumNumberOfProcesses: int(),
+ maximumNumberOfProcesses: anyOf(int(), 'auto'),
minimumNumberOfJobsPerProcess: int(),
buffer: int(),
loadLimit: schema(float(), nullable())
diff --git a/src/Diagnose/SystemResourcesDiagnoseExtension.php b/src/Diagnose/SystemResourcesDiagnoseExtension.php
new file mode 100644
index 00000000000..23db682a88e
--- /dev/null
+++ b/src/Diagnose/SystemResourcesDiagnoseExtension.php
@@ -0,0 +1,41 @@
+writeLineFormatted('System resources:');
+ $output->writeLineFormatted(sprintf('Detected CPU cores: %d', $this->cpuCoreCounter->getDetectedNumberOfCpuCores()));
+
+ $quota = $this->systemResources->getCpuQuota();
+ $output->writeLineFormatted(sprintf(
+ 'cgroup CPU quota: %s',
+ $quota === null ? 'none' : sprintf('%d cores', $quota),
+ ));
+
+ $output->writeLineFormatted(sprintf('Usable CPU cores: %d', $this->cpuCoreCounter->getNumberOfCpuCores()));
+ $output->writeLineFormatted('');
+ }
+
+}
diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php
index 50760e65b92..312ce19d0ea 100644
--- a/src/Parallel/Scheduler.php
+++ b/src/Parallel/Scheduler.php
@@ -19,19 +19,21 @@
final class Scheduler implements DiagnoseExtension
{
- /** @var array{int, int, int, int}|null */
+ public const AUTO = 'auto';
+
+ /** @var array{int, int, int, int, string}|null */
private ?array $storedData = null;
/**
* @param positive-int $jobSize
- * @param positive-int $maximumNumberOfProcesses
+ * @param positive-int|self::AUTO $maximumNumberOfProcesses
* @param positive-int $minimumNumberOfJobsPerProcess
*/
public function __construct(
#[AutowiredParameter(ref: '%parallel.jobSize%')]
private int $jobSize,
#[AutowiredParameter(ref: '%parallel.maximumNumberOfProcesses%')]
- private int $maximumNumberOfProcesses,
+ private int|string $maximumNumberOfProcesses,
#[AutowiredParameter(ref: '%parallel.minimumNumberOfJobsPerProcess%')]
private int $minimumNumberOfJobsPerProcess,
)
@@ -78,25 +80,46 @@ public function scheduleWork(
$cpuCores,
);
- $usedNumberOfProcesses = min($numberOfProcesses, $this->maximumNumberOfProcesses);
- $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses];
+ [$maximumNumberOfProcesses, $decision] = $this->resolveMaximumNumberOfProcesses($cpuCores);
+ $usedNumberOfProcesses = min($numberOfProcesses, $maximumNumberOfProcesses);
+ $this->storedData = [$cpuCores, count($files), count($jobs), $usedNumberOfProcesses, $decision];
return new Schedule($usedNumberOfProcesses, $jobs);
}
+ /**
+ * How many workers may run at once, and a human-readable account of why - which
+ * `diagnose` prints, because a user who thinks the number is wrong needs to see
+ * which input produced it.
+ *
+ * @return array{positive-int, string}
+ */
+ private function resolveMaximumNumberOfProcesses(int $cpuCores): array
+ {
+ if ($this->maximumNumberOfProcesses !== self::AUTO) {
+ return [$this->maximumNumberOfProcesses, 'configured'];
+ }
+
+ return [
+ max(1, $cpuCores),
+ sprintf('auto, limited by %d usable CPU cores', $cpuCores),
+ ];
+ }
+
public function print(Output $output): void
{
if ($this->storedData === null) {
return;
}
- [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses] = $this->storedData;
+ [$cpuCores, $filesCount, $jobsCount, $usedNumberOfProcesses, $decision] = $this->storedData;
$output->writeLineFormatted('Parallel processing scheduler:');
$output->writeLineFormatted(sprintf('# of detected CPU cores: %d', $cpuCores));
$output->writeLineFormatted(sprintf('# of analysed files: %d', $filesCount));
$output->writeLineFormatted(sprintf('# of jobs: %d', $jobsCount));
$output->writeLineFormatted(sprintf('# of spawned processes: %d', $usedNumberOfProcesses));
+ $output->writeLineFormatted(sprintf('Process limit: %s', $decision));
$output->writeLineFormatted('');
}
diff --git a/src/Process/CpuCoreCounter.php b/src/Process/CpuCoreCounter.php
index 49f8c8cb253..7e4e9fff1c9 100644
--- a/src/Process/CpuCoreCounter.php
+++ b/src/Process/CpuCoreCounter.php
@@ -6,6 +6,7 @@
use Fidry\CpuCoreCounter\NumberOfCpuCoreNotFound;
use PHPStan\DependencyInjection\AutowiredParameter;
use PHPStan\DependencyInjection\AutowiredService;
+use function min;
#[AutowiredService]
final class CpuCoreCounter
@@ -13,26 +14,53 @@ final class CpuCoreCounter
private ?int $count = null;
+ private ?int $detectedCount = null;
+
public function __construct(
#[AutowiredParameter(ref: '%parallel.loadLimit%')]
private ?float $loadLimit,
+ private SystemResources $systemResources,
)
{
}
+ /**
+ * Cores PHPStan may actually use: what the machine reports, capped by the CPU
+ * quota of the cgroup it runs in.
+ */
public function getNumberOfCpuCores(): int
{
if ($this->count !== null) {
return $this->count;
}
+ $count = $this->getDetectedNumberOfCpuCores();
+
+ // fidry/cpu-core-counter has no cgroup finder, and its nproc-based default
+ // honours a cpuset affinity mask but not a CFS bandwidth quota, so inside a
+ // `docker run --cpus=2` container it reports the host's core count
+ $quota = $this->systemResources->getCpuQuota();
+ if ($quota !== null) {
+ $count = min($count, $quota);
+ }
+
+ return $this->count = $count;
+ }
+
+ /** What the machine reports before any cgroup quota is applied. */
+ public function getDetectedNumberOfCpuCores(): int
+ {
+ if ($this->detectedCount !== null) {
+ return $this->detectedCount;
+ }
+
try {
- $this->count = (new FidryCpuCoreCounter())->getAvailableForParallelisation(0, null, $this->loadLimit)->availableCpus;
+ $this->detectedCount = (new FidryCpuCoreCounter())->getAvailableForParallelisation(0, null, $this->loadLimit)->availableCpus;
} catch (NumberOfCpuCoreNotFound) {
- $this->count = 1;
+ $this->detectedCount = 1;
}
- return $this->count;
+ return $this->detectedCount;
}
}
diff --git a/src/Process/SystemResources.php b/src/Process/SystemResources.php
new file mode 100644
index 00000000000..5f04485c204
--- /dev/null
+++ b/src/Process/SystemResources.php
@@ -0,0 +1,246 @@
+
+ */
+ private array $cgroupPaths = [];
+
+ /**
+ * @param string $filesystemRoot Prefix for every /proc and /sys path read, so tests
+ * can run against a fixture tree. Empty means the real
+ * filesystem.
+ */
+ public function __construct(private string $filesystemRoot = '')
+ {
+ }
+
+ /**
+ * Number of CPU cores the current cgroup's CFS quota allows, or null when no
+ * cgroup limits CPU bandwidth.
+ *
+ * @return positive-int|null
+ */
+ public function getCpuQuota(): ?int
+ {
+ $quotas = [];
+ foreach ([$this->getCgroupV2CpuQuota(), $this->getCgroupV1CpuQuota()] as $quota) {
+ if ($quota === null) {
+ continue;
+ }
+
+ $quotas[] = $quota;
+ }
+
+ if (count($quotas) === 0) {
+ return null;
+ }
+
+ // a sub-core quota still lets a single worker run, just throttled
+ return max(1, min($quotas));
+ }
+
+ /** @return positive-int|null */
+ private function getCgroupV2CpuQuota(): ?int
+ {
+ $cgroupPath = $this->getCgroupPath('');
+ if ($cgroupPath === null) {
+ return null;
+ }
+
+ $quotas = [];
+ foreach ($this->getAncestorPaths($cgroupPath) as $path) {
+ $cpuMax = $this->readFile('/sys/fs/cgroup' . $path . '/cpu.max');
+ if ($cpuMax === null) {
+ // the cpu controller is not enabled at this depth - the root cgroup
+ // never has the file and a leaf often does not either - which says
+ // nothing about the ancestors that may still carry a quota
+ continue;
+ }
+
+ $parts = explode(' ', trim($cpuMax));
+ if (count($parts) !== 2 || !ctype_digit($parts[0]) || !ctype_digit($parts[1])) {
+ // "max " is how an unlimited cgroup states it
+ continue;
+ }
+
+ $period = (int) $parts[1];
+ if ($period <= 0) {
+ continue;
+ }
+
+ $quotas[] = (int) ceil((int) $parts[0] / $period);
+ }
+
+ return count($quotas) === 0 ? null : max(1, min($quotas));
+ }
+
+ /** @return positive-int|null */
+ private function getCgroupV1CpuQuota(): ?int
+ {
+ $cgroupPath = $this->getCgroupPath('cpu');
+ if ($cgroupPath === null) {
+ return null;
+ }
+
+ $quotas = [];
+ foreach ($this->getAncestorPaths($cgroupPath) as $path) {
+ foreach (['cpu', 'cpu,cpuacct'] as $controllerDir) {
+ $base = '/sys/fs/cgroup/' . $controllerDir . $path;
+ $quota = $this->readIntFile($base . '/cpu.cfs_quota_us');
+ $period = $this->readIntFile($base . '/cpu.cfs_period_us');
+ if ($quota === null || $period === null || $quota <= 0 || $period <= 0) {
+ // -1 means unlimited
+ continue;
+ }
+
+ $quotas[] = (int) ceil($quota / $period);
+ }
+ }
+
+ return count($quotas) === 0 ? null : max(1, min($quotas));
+ }
+
+ /**
+ * The current process' path within a cgroup hierarchy, or null when it is not in
+ * one. An empty controller asks for the v2 unified hierarchy.
+ */
+ private function getCgroupPath(string $controller): ?string
+ {
+ if (array_key_exists($controller, $this->cgroupPaths)) {
+ return $this->cgroupPaths[$controller];
+ }
+
+ return $this->cgroupPaths[$controller] = $this->findCgroupPath($controller);
+ }
+
+ private function findCgroupPath(string $controller): ?string
+ {
+ $contents = $this->readFile('/proc/self/cgroup');
+ if ($contents === null) {
+ return null;
+ }
+
+ foreach (explode("\n", $contents) as $line) {
+ // hierarchy-ID:controller-list:path
+ $parts = explode(':', trim($line), 3);
+ if (count($parts) !== 3) {
+ continue;
+ }
+
+ [, $controllers, $path] = $parts;
+ if ($controller === '') {
+ if ($controllers !== '') {
+ continue;
+ }
+ } elseif (
+ $controllers !== $controller
+ && !str_starts_with($controllers, $controller . ',')
+ && !str_contains($controllers, ',' . $controller)
+ ) {
+ continue;
+ }
+
+ return $path === '/' ? '' : $path;
+ }
+
+ return null;
+ }
+
+ /**
+ * The cgroup's own path and every ancestor up to the root, because a limit set on
+ * an ancestor binds the leaf just as tightly - Kubernetes puts the pod's quota on
+ * the pod slice, not on the container's own cgroup.
+ *
+ * @return list
+ */
+ private function getAncestorPaths(string $path): array
+ {
+ $segments = [];
+ foreach (explode('/', $path) as $segment) {
+ if ($segment === '') {
+ continue;
+ }
+
+ $segments[] = $segment;
+ }
+
+ $paths = [''];
+ for ($i = 1; $i <= count($segments); $i++) {
+ $paths[] = '/' . implode('/', array_slice($segments, 0, $i));
+ }
+
+ return $paths;
+ }
+
+ private function readIntFile(string $path): ?int
+ {
+ $contents = $this->readFile($path);
+ if ($contents === null) {
+ return null;
+ }
+
+ $contents = trim($contents);
+ $negative = str_starts_with($contents, '-');
+ $digits = $negative ? substr($contents, 1) : $contents;
+ if ($digits === '' || !ctype_digit($digits)) {
+ return null;
+ }
+
+ return $negative ? -(int) $digits : (int) $digits;
+ }
+
+ private function readFile(string $path): ?string
+ {
+ $path = $this->filesystemRoot . $path;
+
+ // container filesystems routinely have these present but unreadable, and a
+ // warning from a probe would be worse than not knowing
+ if (!@is_file($path)) {
+ return null;
+ }
+
+ $contents = @file_get_contents($path);
+
+ return $contents === false ? null : $contents;
+ }
+
+}
diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php
index a5fa5b829d6..727d6e0a984 100644
--- a/tests/PHPStan/Parallel/SchedulerTest.php
+++ b/tests/PHPStan/Parallel/SchedulerTest.php
@@ -165,4 +165,32 @@ public function testEveryFileIsScheduledExactlyOnce(): void
}
}
+ public function testAutoUsesAllUsableCores(): void
+ {
+ // 12 usable cores, plenty of jobs - auto follows the cores, not the old
+ // fixed default of 8
+ $scheduler = new Scheduler(1, Scheduler::AUTO, 1);
+ $schedule = $scheduler->scheduleWork(12, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0);
+
+ $this->assertSame(12, $schedule->getNumberOfProcesses());
+ }
+
+ public function testAutoIsStillCappedByTheJobCount(): void
+ {
+ // 40 files -> 2 jobs at size 20, at least 2 jobs per process -> a single
+ // worker no matter how many cores the machine has
+ $scheduler = new Scheduler(20, Scheduler::AUTO, 2);
+ $schedule = $scheduler->scheduleWork(32, array_fill(0, 40, 'file.php'), static fn (string $file): int => 0);
+
+ $this->assertSame(1, $schedule->getNumberOfProcesses());
+ }
+
+ public function testAnExplicitLimitStillWins(): void
+ {
+ $scheduler = new Scheduler(1, 20, 1);
+ $schedule = $scheduler->scheduleWork(32, array_fill(0, 200, 'file.php'), static fn (string $file): int => 0);
+
+ $this->assertSame(20, $schedule->getNumberOfProcesses());
+ }
+
}
diff --git a/tests/PHPStan/Process/SystemResourcesTest.php b/tests/PHPStan/Process/SystemResourcesTest.php
new file mode 100644
index 00000000000..204acd37537
--- /dev/null
+++ b/tests/PHPStan/Process/SystemResourcesTest.php
@@ -0,0 +1,208 @@
+ */
+ private array $roots = [];
+
+ #[Override]
+ protected function tearDown(): void
+ {
+ foreach ($this->roots as $root) {
+ self::removeDirectory($root);
+ }
+
+ $this->roots = [];
+ }
+
+ /**
+ * @return iterable, int|null}>
+ */
+ public static function dataCpuQuota(): iterable
+ {
+ yield 'v2, quota on the leaf' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "200000 100000\n",
+ ],
+ 2,
+ ];
+
+ yield 'v2, quota only on an ancestor' => [
+ [
+ '/proc/self/cgroup' => "0::/foo/bar\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "200000 100000\n",
+ ],
+ 2,
+ ];
+
+ yield 'v2, nested with a tighter ancestor' => [
+ [
+ '/proc/self/cgroup' => "0::/foo/bar\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "100000 100000\n",
+ '/sys/fs/cgroup/foo/bar/cpu.max' => "400000 100000\n",
+ ],
+ 1,
+ ];
+
+ yield 'v2, unlimited' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "max 100000\n",
+ ],
+ null,
+ ];
+
+ yield 'v2, cpu controller not enabled at any level' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/memory.max' => "4294967296\n",
+ ],
+ null,
+ ];
+
+ yield 'v2, quota is not a whole number of cores' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "250000 100000\n",
+ ],
+ 3,
+ ];
+
+ yield 'v2, quota below a single core' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "50000 100000\n",
+ ],
+ 1,
+ ];
+
+ yield 'v2, malformed' => [
+ [
+ '/proc/self/cgroup' => "0::/foo\n",
+ '/sys/fs/cgroup/foo/cpu.max' => "garbage\n",
+ ],
+ null,
+ ];
+
+ yield 'v1, quota' => [
+ [
+ '/proc/self/cgroup' => "4:cpu,cpuacct:/foo\n",
+ '/sys/fs/cgroup/cpu/foo/cpu.cfs_quota_us' => "200000\n",
+ '/sys/fs/cgroup/cpu/foo/cpu.cfs_period_us' => "100000\n",
+ ],
+ 2,
+ ];
+
+ yield 'v1, unlimited' => [
+ [
+ '/proc/self/cgroup' => "4:cpu,cpuacct:/foo\n",
+ '/sys/fs/cgroup/cpu/foo/cpu.cfs_quota_us' => "-1\n",
+ '/sys/fs/cgroup/cpu/foo/cpu.cfs_period_us' => "100000\n",
+ ],
+ null,
+ ];
+
+ yield 'v1, controller mounted as cpu,cpuacct' => [
+ [
+ '/proc/self/cgroup' => "4:cpu,cpuacct:/foo\n",
+ '/sys/fs/cgroup/cpu,cpuacct/foo/cpu.cfs_quota_us' => "400000\n",
+ '/sys/fs/cgroup/cpu,cpuacct/foo/cpu.cfs_period_us' => "100000\n",
+ ],
+ 4,
+ ];
+
+ yield 'no cgroup filesystem at all' => [[], null];
+
+ yield 'in the root cgroup, which has no cpu.max' => [
+ ['/proc/self/cgroup' => "0::/\n"],
+ null,
+ ];
+ }
+
+ /**
+ * @param array $files
+ */
+ #[DataProvider('dataCpuQuota')]
+ public function testGetCpuQuota(array $files, ?int $expectedQuota): void
+ {
+ $resources = new SystemResources($this->createFixtureRoot($files));
+
+ $this->assertSame($expectedQuota, $resources->getCpuQuota());
+ }
+
+ public function testUsableCoresNeverExceedDetectedCoresOnThisMachine(): void
+ {
+ // the important regression: whatever this machine turns out to be, applying
+ // its quota must narrow the core count rather than invent capacity or
+ // collapse to zero - a probe that guesses low would throttle every run
+ $counter = new CpuCoreCounter(null, new SystemResources());
+
+ $this->assertGreaterThanOrEqual(1, $counter->getNumberOfCpuCores());
+ $this->assertLessThanOrEqual($counter->getDetectedNumberOfCpuCores(), $counter->getNumberOfCpuCores());
+ }
+
+ /**
+ * @param array $files
+ */
+ private function createFixtureRoot(array $files): string
+ {
+ $root = sys_get_temp_dir() . '/phpstan-system-resources-' . uniqid();
+ $this->roots[] = $root;
+
+ foreach ($files as $path => $contents) {
+ $fullPath = $root . $path;
+ $directory = dirname($fullPath);
+ if (!is_dir($directory)) {
+ mkdir($directory, 0777, true);
+ }
+
+ FileWriter::write($fullPath, $contents);
+ }
+
+ if (!is_dir($root)) {
+ mkdir($root, 0777, true);
+ }
+
+ return $root;
+ }
+
+ private static function removeDirectory(string $directory): void
+ {
+ if (!is_dir($directory)) {
+ return;
+ }
+
+ foreach (scandir($directory) ?: [] as $entry) {
+ if ($entry === '.' || $entry === '..') {
+ continue;
+ }
+
+ $path = $directory . '/' . $entry;
+ if (is_dir($path)) {
+ self::removeDirectory($path);
+ } else {
+ unlink($path);
+ }
+ }
+
+ rmdir($directory);
+ }
+
+}