diff --git a/src/Cache/AnalysisResultCache.php b/src/Cache/AnalysisResultCache.php index 9c3e5ab0..4cf7e58b 100644 --- a/src/Cache/AnalysisResultCache.php +++ b/src/Cache/AnalysisResultCache.php @@ -64,6 +64,12 @@ final class AnalysisResultCache */ private const METADATA_FILE = '_metadata.json'; + /** + * Stat signatures and content hashes from earlier runs, so an unchanged + * file is not read again just to hash it. + */ + private const FINGERPRINTS_FILE = '_fingerprints.json'; + /** * Format version of the analysis-node payload files. Bump it whenever * their shape, naming, or reference semantics change. It is recorded in @@ -87,8 +93,11 @@ public function __construct( private readonly ComposerJsonProvider $composerJsonProvider = new ComposerJsonProvider(), ) { $this->cacheDirectory = CachePathFactory::getPath($cacheDirectory, $basePath); - $composerFile = rtrim($basePath, '/') . '/composer.json'; - $this->composerHash = $fileHashProvider->hash($composerFile); + $fingerprints = $this->readPath($this->cacheDirectory . '/' . self::FINGERPRINTS_FILE); + $fileHashProvider->useFingerprints($fingerprints ?? []); + + $composerFile = rtrim($basePath, '/') . '/composer.json'; + $this->composerHash = $fileHashProvider->hash($composerFile); } /** @@ -96,6 +105,10 @@ public function __construct( */ public function load(string $key, array $metadata): ?RuleViolationCollection { + // the metadata is built after every file has been hashed, so the + // fingerprints recorded while hashing are complete by now + $this->storeFileFingerprints(); + $payload = $this->read($key); if ($payload === null) { @@ -156,6 +169,7 @@ public function clear(): void // with other files keeps them: // // - _metadata.json + // - _fingerprints.json // - 8679959e514dd177bc6fd1830ee55b48.json violation payloads // - analysis-nodes-0a489b7052209b1aa6ce9a7c07ce0627.json per-file payloads (0.17.0+) // - class-nodes-0a489b7052209b1aa6ce9a7c07ce0627.json per-file payloads (pre 0.17.0: to be cleared later) @@ -168,6 +182,7 @@ public function clear(): void $filename = basename($path); if ( $filename === self::METADATA_FILE + || $filename === self::FINGERPRINTS_FILE || preg_match( '/^(?:(?:analysis-nodes-|class-nodes-)?[a-f0-9]{32}\.json|structarmed-worker-.+)$/', $filename, @@ -218,6 +233,25 @@ public function shouldInvalidate(): bool || ($payload['composerHash'] ?? null) !== $this->composerHash; } + /** + * Writes nothing when every hash came from a stored fingerprint. + */ + private function storeFileFingerprints(): void + { + $fingerprints = $this->fileHashProvider->newFingerprints(); + + if ($fingerprints === null) { + return; + } + + $this->ensureCacheInitialised(); + + file_put_contents( + $this->cacheDirectory . '/' . self::FINGERPRINTS_FILE, + json_encode($fingerprints, JSON_INVALID_UTF8_SUBSTITUTE | JSON_THROW_ON_ERROR) + ); + } + private function ensureCacheInitialised(): void { if ($this->isCacheInitialised) { diff --git a/src/Cache/FileHashProvider.php b/src/Cache/FileHashProvider.php index 0b1bca4a..ca0425f8 100644 --- a/src/Cache/FileHashProvider.php +++ b/src/Cache/FileHashProvider.php @@ -7,31 +7,108 @@ use function array_fill_keys; use function array_intersect_key; use function hash_file; +use function is_string; +use function stat; +use function substr; +use function time; + +use const PHP_OS_FAMILY; /** * Provides memoised content hashes for files that are treated as immutable * during a single analysis run. + * + * Hashes from a previous run can be reused through fingerprints: a file whose + * stat signature is unchanged keeps its stored hash without being read again. */ final class FileHashProvider { /** @var array */ private array $hashes = []; + /** + * Per file, the 32-character content hash and the stat signature it was + * taken at, joined as "hash:signature": one string per file instead of a + * pair keeps the map several MB smaller on large projects. + * + * @var array + */ + private array $fingerprints = []; + + private bool $hasNewFingerprints = false; + + /** + * A file changed within the run's starting second could change again in + * that second without changing its signature, so only a file whose ctime + * is older is fingerprinted. Windows reports the creation time as ctime, + * so a same-size rewrite that keeps the mtime would go unnoticed there. + */ + private readonly int $fingerprintBefore; + + public function __construct() + { + $this->fingerprintBefore = PHP_OS_FAMILY === 'Windows' ? 0 : time(); + } + public function hash(string $file): string { if (isset($this->hashes[$file])) { return $this->hashes[$file]; } + $stat = @stat($file); + + if ($stat === false) { + return ''; + } + + $signature = $stat['mtime'] . ':' . $stat['ctime'] . ':' . $stat['size'] . ':' . $stat['ino']; + $fingerprint = $this->fingerprints[$file] ?? null; + + if ($fingerprint !== null && substr($fingerprint, 33) === $signature) { + return $this->hashes[$file] = substr($fingerprint, 0, 32); + } + $hash = @hash_file('xxh128', $file); if ($hash === false) { return ''; } + if ($stat['ctime'] < $this->fingerprintBefore) { + $this->fingerprints[$file] = $hash . ':' . $signature; + $this->hasNewFingerprints = true; + } + return $this->hashes[$file] = $hash; } + /** + * @param array $fingerprints + */ + public function useFingerprints(array $fingerprints): void + { + foreach ($fingerprints as $file => $fingerprint) { + if (is_string($fingerprint)) { + $this->fingerprints[$file] = $fingerprint; + } + } + } + + /** + * @return array|null null when nothing new was fingerprinted + */ + public function newFingerprints(): ?array + { + if (! $this->hasNewFingerprints) { + return null; + } + + $this->hasNewFingerprints = false; + + return $this->fingerprints; + } + /** * @param list $files */ @@ -45,6 +122,8 @@ public function forFiles(array $files): self public function clear(): void { - $this->hashes = []; + $this->hashes = []; + $this->fingerprints = []; + $this->hasNewFingerprints = false; } } diff --git a/tests/Cache/AnalysisResultCacheTest.php b/tests/Cache/AnalysisResultCacheTest.php index 2ecfd902..c323fa9a 100644 --- a/tests/Cache/AnalysisResultCacheTest.php +++ b/tests/Cache/AnalysisResultCacheTest.php @@ -49,6 +49,7 @@ use function preg_match; use function random_bytes; use function rmdir; +use function stat; use function str_replace; use function str_starts_with; use function sys_get_temp_dir; @@ -56,6 +57,7 @@ use function unlink; use const JSON_THROW_ON_ERROR; +use const PHP_OS_FAMILY; #[CoversClass(AnalysisCacheMetadataFactory::class)] #[CoversClass(AnalysisResultCache::class)] @@ -362,6 +364,7 @@ public function testClearRemovesOnlyCacheFilesAndKeepsDirectoryWithOtherEntries( $hash = hash('xxh128', 'key'); $cacheFiles = [ '_metadata.json', + '_fingerprints.json', $hash . '.json', 'analysis-nodes-' . $hash . '.json', 'class-nodes-' . $hash . '.json', @@ -389,6 +392,56 @@ public function testClearRemovesOnlyCacheFilesAndKeepsDirectoryWithOtherEntries( } } + public function testReusesStoredFileFingerprintsAndStoresNewOnesOnLoad(): void + { + $cacheDirectory = $this->createTempDirectory(); + $sourceFile = $cacheDirectory . '/Foo.php'; + + file_put_contents($sourceFile, 'assertIsArray($stat); + + $fingerprint = hash('xxh128', 'stored') . ':' + . $stat['mtime'] . ':' . $stat['ctime'] . ':' . $stat['size'] . ':' . $stat['ino']; + + file_put_contents($cacheDirectory . '/_fingerprints.json', json_encode([ + $sourceFile => $fingerprint, + ], JSON_THROW_ON_ERROR)); + + try { + $fileHashProvider = new FileHashProvider(); + $analysisResultCache = new AnalysisResultCache(__DIR__, $fileHashProvider, $cacheDirectory); + + $this->assertSame(hash('xxh128', 'stored'), $fileHashProvider->hash($sourceFile)); + + // this test file was last changed before the run started, so it is fingerprinted + $fileHashProvider->hash(__FILE__); + + $this->assertNull($analysisResultCache->load(hash('xxh128', 'key'), [])); + + $fingerprints = json_decode( + (string) file_get_contents($cacheDirectory . '/_fingerprints.json'), + true, + flags: JSON_THROW_ON_ERROR + ); + + $this->assertIsArray($fingerprints); + $this->assertSame($fingerprint, $fingerprints[$sourceFile]); + $this->assertSame(PHP_OS_FAMILY !== 'Windows', isset($fingerprints[__FILE__])); + + // nothing new was fingerprinted since, so nothing is written again + unlink($cacheDirectory . '/_fingerprints.json'); + + $this->assertNull($analysisResultCache->load(hash('xxh128', 'key'), [])); + + $this->assertFileDoesNotExist($cacheDirectory . '/_fingerprints.json'); + } finally { + unlink($sourceFile); + $this->removeTempDirectory($cacheDirectory); + } + } + public function testCacheMetadataIsNotDifferentWhenCacheDirectoryIsMissing(): void { $cacheDirectory = $this->createTempDirectory(); diff --git a/tests/Cache/FileHashProviderTest.php b/tests/Cache/FileHashProviderTest.php index b4ff2eef..43d9e15c 100644 --- a/tests/Cache/FileHashProviderTest.php +++ b/tests/Cache/FileHashProviderTest.php @@ -10,7 +10,11 @@ use PHPUnit\Framework\TestCase; use function file_put_contents; +use function hash; use function hash_file; +use function stat; + +use const PHP_OS_FAMILY; #[CoversClass(FileHashProvider::class)] final class FileHashProviderTest extends TestCase @@ -50,6 +54,8 @@ public function testDoesNotMemoiseFailedHash(): void $fileHashProvider = new FileHashProvider(); $this->assertSame('', $fileHashProvider->hash($file)); + // stat() succeeds on a directory, but it cannot be read to be hashed + $this->assertSame('', $fileHashProvider->hash($directory)); file_put_contents($file, 'assertSame(hash_file('xxh128', $unrelatedFile), $providerForFiles->hash($unrelatedFile)); $this->assertSame(hash_file('xxh128', $missingFile), $providerForFiles->hash($missingFile)); } + + public function testUsesFingerprintedHashOnlyWhileStatSignatureIsUnchanged(): void + { + $unchangedFile = $this->makeTemporaryFile('structarmed-file-hash'); + $changedFile = $this->makeTemporaryFile('structarmed-file-hash'); + $malformedFile = $this->makeTemporaryFile('structarmed-file-hash'); + + file_put_contents($unchangedFile, 'useFingerprints([ + $unchangedFile => $storedHash . ':' . $this->signature($unchangedFile), + $changedFile => $storedHash . ':1:1:1:1', + $malformedFile => [$storedHash, $this->signature($malformedFile)], + ]); + + $this->assertSame($storedHash, $fileHashProvider->hash($unchangedFile)); + $this->assertSame(hash_file('xxh128', $changedFile), $fileHashProvider->hash($changedFile)); + $this->assertSame(hash_file('xxh128', $malformedFile), $fileHashProvider->hash($malformedFile)); + } + + public function testFingerprintsOnlyFilesChangedBeforeTheRunStarted(): void + { + $fileHashProvider = new FileHashProvider(); + $freshFile = $this->makeTemporaryFile('structarmed-file-hash'); + + file_put_contents($freshFile, 'hash($freshFile); + + $this->assertNull($fileHashProvider->newFingerprints()); + + // this test file was last changed before the run started + $fileHashProvider->hash(__FILE__); + + $fingerprint = hash_file('xxh128', __FILE__) . ':' . $this->signature(__FILE__); + + $this->assertSame( + PHP_OS_FAMILY === 'Windows' ? null : [__FILE__ => $fingerprint], + $fileHashProvider->newFingerprints() + ); + $this->assertNull($fileHashProvider->newFingerprints()); + + $fileHashProvider->clear(); + $fileHashProvider->hash(__FILE__); + $fileHashProvider->clear(); + + $this->assertNull($fileHashProvider->newFingerprints()); + } + + private function signature(string $file): string + { + $stat = stat($file); + $this->assertIsArray($stat); + + return $stat['mtime'] . ':' . $stat['ctime'] . ':' . $stat['size'] . ':' . $stat['ino']; + } }