From 85e94945930ccb4d24acdcf199148c90516f82f3 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Wed, 22 Jul 2026 19:57:16 +0000 Subject: [PATCH 1/8] Split registry read and write effects Amp-Thread-ID: https://ampcode.com/threads/T-019f8b54-076c-701a-863f-75106cd6f5bd Co-authored-by: Thomas Honeyman --- app-e2e/src/Test/E2E/Scripts.purs | 6 +- app/src/App/API.purs | 6 +- app/src/App/Effect/PackageSets.purs | 6 +- app/src/App/Effect/Registry.purs | 143 +++++++++++++++----------- app/src/App/Server/Env.purs | 22 ++-- app/src/App/Server/JobExecutor.purs | 4 +- app/src/App/Server/MatrixBuilder.purs | 10 +- app/test/App/API.purs | 2 +- app/test/App/Effect/Registry.purs | 10 +- app/test/Test/Assert/Run.purs | 62 ++++++----- scripts/src/DailyImporter.purs | 6 +- scripts/src/PackageSetUpdater.purs | 6 +- scripts/src/PackageTransferrer.purs | 6 +- scripts/src/VerifyIntegrity.purs | 4 +- 14 files changed, 165 insertions(+), 128 deletions(-) diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index 69d8f6be..72a05661 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -191,7 +191,7 @@ runDailyImporterScript = do result <- liftAff $ DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv @@ -207,7 +207,7 @@ runPackageTransferrerScript = do result <- liftAff $ PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv @@ -223,7 +223,7 @@ runPackageSetUpdaterScript = do result <- liftAff $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv diff --git a/app/src/App/API.purs b/app/src/App/API.purs index d8d80e77..a4b69c5d 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -76,7 +76,7 @@ import Registry.App.Effect.PackageSets (Change(..), PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY, REGISTRY_READ) import Registry.App.Effect.Registry as ManifestIndex import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) @@ -349,7 +349,7 @@ resolveCompilerAndDeps -> Manifest -> Maybe Version -- payload.compiler -> Maybe (Map PackageName Version) -- payload.resolutions - -> Run (REGISTRY + LOG + AFF + EXCEPT String + r) { compiler :: Version, resolutions :: Map PackageName Version } + -> Run (LOG + AFF + EXCEPT String + r) { compiler :: Version, resolutions :: Map PackageName Version } resolveCompilerAndDeps compilerIndex manifest@(Manifest { dependencies }) maybeCompiler maybeResolutions = do Log.debug "Resolving compiler and dependencies..." case maybeCompiler of @@ -1067,7 +1067,7 @@ type FindAllCompilersResult = findAllCompilers :: forall r . { source :: FilePath, manifest :: Manifest, compilers :: NonEmptyArray Version } - -> Run (REGISTRY + STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult + -> Run (REGISTRY_READ + STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult findAllCompilers { source, manifest, compilers } = do compilerIndex <- MatrixBuilder.readCompilerIndex checkedCompilers <- for compilers \target -> do diff --git a/app/src/App/Effect/PackageSets.purs b/app/src/App/Effect/PackageSets.purs index f50f7344..9309f248 100644 --- a/app/src/App/Effect/PackageSets.purs +++ b/app/src/App/Effect/PackageSets.purs @@ -24,7 +24,7 @@ import Registry.App.CLI.Purs as Purs import Registry.App.CLI.Tar as Tar import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Storage (STORAGE) import Registry.App.Effect.Storage as Storage @@ -86,7 +86,7 @@ type PackageSetsEnv = -- | A handler for the PACKAGE_SETS effect which compiles the package sets and -- | returns the results. -handle :: forall r a. PackageSetsEnv -> PackageSets a -> Run (REGISTRY + STORAGE + LOG + AFF + EFFECT + r) a +handle :: forall r a. PackageSetsEnv -> PackageSets a -> Run (REGISTRY_READ + STORAGE + LOG + AFF + EFFECT + r) a handle env = case _ of UpgradeAtomic oldSet@(PackageSet { packages }) compiler changes reply -> reply <$> Except.runExcept do Log.info $ "Performing atomic upgrade of package set " <> Version.print (un PackageSet oldSet).version @@ -423,7 +423,7 @@ type ValidatedCandidates = } -- | Validate a package set is self-contained, ignoring version bounds. -validatePackageSet :: forall r. PackageSet -> Run (REGISTRY + LOG + EXCEPT String + r) Unit +validatePackageSet :: forall r. PackageSet -> Run (REGISTRY_READ + LOG + EXCEPT String + r) Unit validatePackageSet (PackageSet set) = do Log.debug $ "Validating package set version " <> Version.print set.version index <- Registry.readAllManifests diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index d3c0acb9..7ea05e47 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -68,74 +68,91 @@ type REGISTRY_CACHE r = (registryCache :: Cache RegistryCache | r) _registryCache :: Proxy "registryCache" _registryCache = Proxy -data Registry a +data RegistryRead a = ReadManifest PackageName Version (Either String (Maybe Manifest) -> a) - | WriteManifest Manifest (Either String Unit -> a) - | DeleteManifest PackageName Version (Either String Unit -> a) | ReadAllManifests (Either String ManifestIndex -> a) | ReadMetadata PackageName (Either String (Maybe Metadata) -> a) - | WriteMetadata PackageName Metadata (Either String Unit -> a) | ReadAllMetadata (Either String (Map PackageName Metadata) -> a) | ReadLatestPackageSet (Either String (Maybe PackageSet) -> a) - | WritePackageSet PackageSet String (Either String Unit -> a) | ReadAllPackageSets (Either String (Map Version PackageSet) -> a) + +derive instance Functor RegistryRead + +data RegistryWrite a + = WriteManifest Manifest (Either String Unit -> a) + | DeleteManifest PackageName Version (Either String Unit -> a) + | WriteMetadata PackageName Metadata (Either String Unit -> a) + | WritePackageSet PackageSet String (Either String Unit -> a) | MirrorPackageSet PackageSet (Either String Unit -> a) -derive instance Functor Registry +derive instance Functor RegistryWrite + +-- | An effect for reading registry resources, like manifests, metadata, and +-- | package sets. +type REGISTRY_READ r = (registryRead :: RegistryRead | r) + +_registryRead :: Proxy "registryRead" +_registryRead = Proxy + +-- | An effect for writing registry resources, like manifests, metadata, and +-- | package sets. +type REGISTRY_WRITE r = (registryWrite :: RegistryWrite | r) --- | An effect for interacting with registry resources, like manifests, metadata, --- | and the package sets. -type REGISTRY r = (registry :: Registry | r) +_registryWrite :: Proxy "registryWrite" +_registryWrite = Proxy -_registry :: Proxy "registry" -_registry = Proxy +-- | An effect for reading and writing registry resources. +type REGISTRY r = (REGISTRY_READ + REGISTRY_WRITE + r) -- | Read a manifest from the manifest index -readManifest :: forall r. PackageName -> Version -> Run (REGISTRY + EXCEPT String + r) (Maybe Manifest) -readManifest name version = Except.rethrow =<< Run.lift _registry (ReadManifest name version identity) +readManifest :: forall r. PackageName -> Version -> Run (REGISTRY_READ + EXCEPT String + r) (Maybe Manifest) +readManifest name version = Except.rethrow =<< Run.lift _registryRead (ReadManifest name version identity) -- | Write a manifest to the manifest index -writeManifest :: forall r. Manifest -> Run (REGISTRY + EXCEPT String + r) Unit -writeManifest manifest = Except.rethrow =<< Run.lift _registry (WriteManifest manifest identity) +writeManifest :: forall r. Manifest -> Run (REGISTRY_WRITE + EXCEPT String + r) Unit +writeManifest manifest = Except.rethrow =<< Run.lift _registryWrite (WriteManifest manifest identity) -- | Delete a manifest from the manifest index -deleteManifest :: forall r. PackageName -> Version -> Run (REGISTRY + EXCEPT String + r) Unit -deleteManifest name version = Except.rethrow =<< Run.lift _registry (DeleteManifest name version identity) +deleteManifest :: forall r. PackageName -> Version -> Run (REGISTRY_WRITE + EXCEPT String + r) Unit +deleteManifest name version = Except.rethrow =<< Run.lift _registryWrite (DeleteManifest name version identity) -- | Read the entire manifest index -readAllManifests :: forall r. Run (REGISTRY + EXCEPT String + r) ManifestIndex -readAllManifests = Except.rethrow =<< Run.lift _registry (ReadAllManifests identity) +readAllManifests :: forall r. Run (REGISTRY_READ + EXCEPT String + r) ManifestIndex +readAllManifests = Except.rethrow =<< Run.lift _registryRead (ReadAllManifests identity) -- | Read the registry metadata for a package -readMetadata :: forall r. PackageName -> Run (REGISTRY + EXCEPT String + r) (Maybe Metadata) -readMetadata name = Except.rethrow =<< Run.lift _registry (ReadMetadata name identity) +readMetadata :: forall r. PackageName -> Run (REGISTRY_READ + EXCEPT String + r) (Maybe Metadata) +readMetadata name = Except.rethrow =<< Run.lift _registryRead (ReadMetadata name identity) -- | Write the registry metadata for a package -writeMetadata :: forall r. PackageName -> Metadata -> Run (REGISTRY + EXCEPT String + r) Unit -writeMetadata name metadata = Except.rethrow =<< Run.lift _registry (WriteMetadata name metadata identity) +writeMetadata :: forall r. PackageName -> Metadata -> Run (REGISTRY_WRITE + EXCEPT String + r) Unit +writeMetadata name metadata = Except.rethrow =<< Run.lift _registryWrite (WriteMetadata name metadata identity) -- | Read the registry metadata for all packages -readAllMetadata :: forall r. Run (REGISTRY + EXCEPT String + r) (Map PackageName Metadata) -readAllMetadata = Except.rethrow =<< Run.lift _registry (ReadAllMetadata identity) +readAllMetadata :: forall r. Run (REGISTRY_READ + EXCEPT String + r) (Map PackageName Metadata) +readAllMetadata = Except.rethrow =<< Run.lift _registryRead (ReadAllMetadata identity) -- | Read the latest package set from the registry -readLatestPackageSet :: forall r. Run (REGISTRY + EXCEPT String + r) (Maybe PackageSet) -readLatestPackageSet = Except.rethrow =<< Run.lift _registry (ReadLatestPackageSet identity) +readLatestPackageSet :: forall r. Run (REGISTRY_READ + EXCEPT String + r) (Maybe PackageSet) +readLatestPackageSet = Except.rethrow =<< Run.lift _registryRead (ReadLatestPackageSet identity) -- | Write a package set to the registry -writePackageSet :: forall r. PackageSet -> String -> Run (REGISTRY + EXCEPT String + r) Unit -writePackageSet set message = Except.rethrow =<< Run.lift _registry (WritePackageSet set message identity) +writePackageSet :: forall r. PackageSet -> String -> Run (REGISTRY_WRITE + EXCEPT String + r) Unit +writePackageSet set message = Except.rethrow =<< Run.lift _registryWrite (WritePackageSet set message identity) -- | Read all package sets from the registry -readAllPackageSets :: forall r. Run (REGISTRY + EXCEPT String + r) (Map Version PackageSet) -readAllPackageSets = Except.rethrow =<< Run.lift _registry (ReadAllPackageSets identity) +readAllPackageSets :: forall r. Run (REGISTRY_READ + EXCEPT String + r) (Map Version PackageSet) +readAllPackageSets = Except.rethrow =<< Run.lift _registryRead (ReadAllPackageSets identity) -- | Mirror a package set to the legacy package-sets repo -mirrorPackageSet :: forall r. PackageSet -> Run (REGISTRY + EXCEPT String + r) Unit -mirrorPackageSet set = Except.rethrow =<< Run.lift _registry (MirrorPackageSet set identity) +mirrorPackageSet :: forall r. PackageSet -> Run (REGISTRY_WRITE + EXCEPT String + r) Unit +mirrorPackageSet set = Except.rethrow =<< Run.lift _registryWrite (MirrorPackageSet set identity) -interpret :: forall r a. (Registry ~> Run r) -> Run (REGISTRY + r) a -> Run r a -interpret handler = Run.interpret (Run.on _registry handler Run.send) +interpretRead :: forall r a. (RegistryRead ~> Run r) -> Run (REGISTRY_READ + r) a -> Run r a +interpretRead handler = Run.interpret (Run.on _registryRead handler Run.send) + +interpretWrite :: forall r a. (RegistryWrite ~> Run r) -> Run (REGISTRY_WRITE + r) a -> Run r a +interpretWrite handler = Run.interpret (Run.on _registryWrite handler Run.send) -- | A legend for repositories that can be fetched and committed to. data RepoKey @@ -213,18 +230,28 @@ defaultRepos = , legacyPackageSets: Legacy.PackageSet.legacyPackageSetsRepo } --- | Handle the REGISTRY effect by downloading the registry and registry-index --- | repositories locally and reading and writing their contents from disk. +data RegistryOperation a + = ReadOperation (RegistryRead a) + | WriteOperation (RegistryWrite a) + +-- | Handle the REGISTRY_READ effect by downloading the registry and +-- | registry-index repositories locally and reading their contents from disk. +handleRead :: forall r a. RegistryEnv -> RegistryRead a -> Run (GITHUB + LOG + AFF + EFFECT + r) a +handleRead env = handleOperation env <<< ReadOperation + +-- | Handle the REGISTRY_WRITE effect by writing registry resources to disk. -- | Writes can optionally commit and push to the upstream Git repository. --- | --- | This handler enforces a memory-only cache: we do not want to cache on the --- | file system or other storage because this handler relies on the registry --- | Git repositories instead. -handle :: forall r a. RegistryEnv -> Registry a -> Run (GITHUB + LOG + AFF + EFFECT + r) a -handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<< case _ of - ReadManifest name version reply -> do +handleWrite :: forall r a. RegistryEnv -> RegistryWrite a -> Run (GITHUB + LOG + AFF + EFFECT + r) a +handleWrite env = handleOperation env <<< WriteOperation + +-- | These handlers share a memory-only cache: we do not want to cache on the +-- | file system or other storage because they rely on the registry Git +-- | repositories instead. +handleOperation :: forall r a. RegistryEnv -> RegistryOperation a -> Run (GITHUB + LOG + AFF + EFFECT + r) a +handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<< case _ of + ReadOperation (ReadManifest name version reply) -> do let formatted = formatPackageVersion name version - handle env (ReadAllManifests identity) >>= case _ of + handleOperation env (ReadOperation (ReadAllManifests identity)) >>= case _ of Left error -> pure $ reply $ Left error Right index -> case ManifestIndex.lookup name version index of Nothing -> do @@ -233,10 +260,10 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Just manifest -> do pure $ reply $ Right $ Just manifest - WriteManifest manifest@(Manifest { name, version }) reply -> map (map reply) Except.runExcept do + WriteOperation (WriteManifest manifest@(Manifest { name, version }) reply) -> map (map reply) Except.runExcept do let formatted = formatPackageVersion name version Log.info $ "Writing manifest for " <> formatted <> ":\n" <> printJson Manifest.codec manifest - index <- Except.rethrow =<< handle env (ReadAllManifests identity) + index <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) case ManifestIndex.insert ManifestIndex.ConsiderRanges manifest index of Left error -> Except.throw $ Array.fold @@ -256,10 +283,10 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Git.Changed -> Log.info "Wrote and committed manifest." Cache.put _registryCache AllManifests updated - DeleteManifest name version reply -> map (map reply) Except.runExcept do + WriteOperation (DeleteManifest name version reply) -> map (map reply) Except.runExcept do let formatted = formatPackageVersion name version Log.info $ "Deleting manifest for " <> formatted - index <- Except.rethrow =<< handle env (ReadAllManifests identity) + index <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) case ManifestIndex.delete ManifestIndex.ConsiderRanges name version index of Left error -> Except.throw $ Array.fold @@ -281,7 +308,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Log.info "Wrote and committed manifest." Cache.put _registryCache AllManifests updated - ReadAllManifests reply -> map (map reply) Except.runExcept do + ReadOperation (ReadAllManifests reply) -> map (map reply) Except.runExcept do let refreshIndex = do let indexPath = repoPath ManifestIndexRepo @@ -303,7 +330,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Log.info "Manifest index has changed, replacing cache..." refreshIndex - ReadMetadata name reply -> map (map reply) Except.runExcept do + ReadOperation (ReadMetadata name reply) -> map (map reply) Except.runExcept do let printedName = PackageName.print name let dir = repoPath RegistryRepo @@ -363,7 +390,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Cache.delete _registryCache AllMetadata readMetadataFromDisk - WriteMetadata name metadata reply -> map (map reply) Except.runExcept do + WriteOperation (WriteMetadata name metadata reply) -> map (map reply) Except.runExcept do let printedName = PackageName.print name Log.info $ "Writing metadata for " <> printedName Log.debug $ printJson Metadata.codec metadata @@ -384,7 +411,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << for_ cache \cached -> Cache.put _registryCache AllMetadata (Map.insert name metadata cached) - ReadAllMetadata reply -> map (map reply) Except.runExcept do + ReadOperation (ReadAllMetadata reply) -> map (map reply) Except.runExcept do let refreshMetadata = do let dir = repoPath RegistryRepo @@ -408,7 +435,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Log.info "Registry repo has changed, replacing metadata cache..." refreshMetadata - ReadLatestPackageSet reply -> map (map reply) Except.runExcept do + ReadOperation (ReadLatestPackageSet reply) -> map (map reply) Except.runExcept do pull RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error Right _ -> pure unit @@ -429,7 +456,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Log.debug $ "Successfully read package set " <> printed pure $ Just set - WritePackageSet set@(PackageSet { version }) message reply -> map (map reply) Except.runExcept do + WriteOperation (WritePackageSet set@(PackageSet { version }) message reply) -> map (map reply) Except.runExcept do pull RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error Right _ -> pure unit @@ -445,7 +472,7 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << Right Git.NoChange -> Log.info "Did not commit package set because it was unchanged." Right Git.Changed -> Log.info "Wrote and committed package set." - ReadAllPackageSets reply -> map (map reply) Except.runExcept do + ReadOperation (ReadAllPackageSets reply) -> map (map reply) Except.runExcept do pull RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error Right _ -> pure unit @@ -469,11 +496,11 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << -- https://github.com/purescript/package-sets/blob/psc-0.15.4-20220829/release.sh -- https://github.com/purescript/package-sets/blob/psc-0.15.4-20220829/update-latest-compatible-sets.sh - MirrorPackageSet set@(PackageSet { version }) reply -> map (map reply) Except.runExcept do + WriteOperation (MirrorPackageSet set@(PackageSet { version }) reply) -> map (map reply) Except.runExcept do let name = Version.print version Log.info $ "Mirroring legacy package set " <> name <> " to the legacy package sets repo" - manifests <- Except.rethrow =<< handle env (ReadAllManifests identity) + manifests <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) Log.debug $ "Converting package set..." converted <- case Legacy.PackageSet.convertPackageSet manifests set of diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index 5c0ddb39..b2310614 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -147,19 +147,19 @@ runEffects env operation = Aff.attempt do writeMode | env.vars.readOnly = Registry.ReadOnly | otherwise = Registry.CommitAs (Git.pacchettibottiCommitter env.vars.token) + registryEnv = + { jobId: env.jobId + , repos: Registry.defaultRepos + , pull: Git.ForceClean + , write: writeMode + , workdir: scratchDir + , debouncer: env.debouncer + , cacheRef: env.registryCacheRef + } operation # PackageSets.interpret (PackageSets.handle { workdir: scratchDir }) - # Registry.interpret - ( Registry.handle - { jobId: env.jobId - , repos: Registry.defaultRepos - , pull: Git.ForceClean - , write: writeMode - , workdir: scratchDir - , debouncer: env.debouncer - , cacheRef: env.registryCacheRef - } - ) + # Registry.interpretWrite (Registry.handleWrite registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # Pursuit.interpret (if env.vars.readOnly then Pursuit.handlePure else Pursuit.handleAff env.vars.token) # Storage.interpret (if env.vars.readOnly then Storage.handleReadOnly env.cacheDir else Storage.handleS3 { s3: { key: env.vars.spacesKey, secret: env.vars.spacesSecret }, cache: env.cacheDir }) # Source.interpret Source.handle diff --git a/app/src/App/Server/JobExecutor.purs b/app/src/App/Server/JobExecutor.purs index 8bcd8d63..fe505cd1 100644 --- a/app/src/App/Server/JobExecutor.purs +++ b/app/src/App/Server/JobExecutor.purs @@ -21,7 +21,7 @@ import Registry.App.Effect.Db (DB) import Registry.App.Effect.Db as Db import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.App.Server.Env (ServerEffects, ServerEnv, runEffects) import Registry.App.Server.MatrixBuilder as MatrixBuilder @@ -186,7 +186,7 @@ executeJob _ = case _ of } PackageSetJob payload -> API.packageSetUpdate payload -upgradeRegistryToNewCompiler :: forall r. Version -> Run (DB + LOG + EXCEPT String + REGISTRY + r) Unit +upgradeRegistryToNewCompiler :: forall r. Version -> Run (DB + LOG + EXCEPT String + REGISTRY_READ + r) Unit upgradeRegistryToNewCompiler newCompilerVersion = do Log.info $ "New compiler found: " <> Version.print newCompilerVersion Log.info "Starting upgrade of the whole registry to the new compiler..." diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index 73941d59..6e1ebc00 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -30,7 +30,7 @@ import Registry.App.CLI.PursVersions as PursVersions import Registry.App.CLI.Tar as Tar import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY, REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Storage (STORAGE) import Registry.App.Effect.Storage as Storage @@ -102,7 +102,7 @@ runMatrixJob { compilerVersion, packageName, packageVersion, payload: buildPlan -- TODO feels like we should be doing this at startup and use the cache instead -- of reading files all over again -readCompilerIndex :: forall r. Run (REGISTRY + AFF + EXCEPT String + r) Solver.CompilerIndex +readCompilerIndex :: forall r. Run (REGISTRY_READ + AFF + EXCEPT String + r) Solver.CompilerIndex readCompilerIndex = do metadata <- Registry.readAllMetadata manifests <- Registry.readAllManifests @@ -137,7 +137,7 @@ installBuildPlan resolutions dependenciesDir = do -- | Convert resolutions (Map PackageName Version) to build plan entries using metadata. -- | Fetches metadata for each package as needed. -resolutionsToBuildPlan :: forall r. Map PackageName Version -> Run (REGISTRY + EXCEPT String + r) (Map PackageName BuildPlanEntry) +resolutionsToBuildPlan :: forall r. Map PackageName Version -> Run (REGISTRY_READ + EXCEPT String + r) (Map PackageName BuildPlanEntry) resolutionsToBuildPlan resolutions = forWithIndex resolutions \name version -> do maybeMetadata <- Registry.readMetadata name @@ -190,7 +190,7 @@ solveForAllCompilers solverData@{ compiler } = do trySolveForCompiler (solverData { compiler = target }) pure $ Set.fromFoldable $ Array.catMaybes newJobs -solveDependantsForCompiler :: forall r. MatrixSolverData -> Run (EXCEPT String + LOG + REGISTRY + r) (Set MatrixSolverResult) +solveDependantsForCompiler :: forall r. MatrixSolverData -> Run (EXCEPT String + LOG + REGISTRY_READ + r) (Set MatrixSolverResult) solveDependantsForCompiler { compilerIndex, name, version, compiler } = do manifestIndex <- Registry.readAllManifests let seed = Tuple name version @@ -276,7 +276,7 @@ trySolveForCompiler { compilerIndex, compiler, name, version, dependencies } = d ] pure Nothing -checkIfNewCompiler :: forall r. Run (EXCEPT String + LOG + REGISTRY + AFF + r) (Maybe Version) +checkIfNewCompiler :: forall r. Run (EXCEPT String + LOG + REGISTRY_READ + AFF + r) (Maybe Version) checkIfNewCompiler = do Log.info "Checking if there's a new compiler in town..." latestCompiler <- NonEmptyArray.foldr1 max <$> PursVersions.pursVersions diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 0db7135d..0ecd69fd 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -78,7 +78,7 @@ assertPublicationState . PackageName -> Version -> { manifest :: Boolean, metadata :: Boolean, storage :: Boolean } - -> Run (Registry.REGISTRY + Storage.STORAGE + Except.EXCEPT String + r) Unit + -> Run (Registry.REGISTRY_READ + Storage.STORAGE + Except.EXCEPT String + r) Unit assertPublicationState name version expected = do storedVersions <- Storage.query name maybeMetadata <- Registry.readMetadata name diff --git a/app/test/App/Effect/Registry.purs b/app/test/App/Effect/Registry.purs index 241ab9af..e9f2f7f0 100644 --- a/app/test/App/Effect/Registry.purs +++ b/app/test/App/Effect/Registry.purs @@ -13,7 +13,7 @@ import Registry.App.Effect.GitHub (GITHUB, GitHub) import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG, Log(..)) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY, RegistryEnv, WriteMode(..)) +import Registry.App.Effect.Registry (REGISTRY_READ, RegistryEnv, WriteMode(..)) import Registry.App.Effect.Registry as Registry import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.Tmp as Tmp @@ -36,7 +36,7 @@ spec = do Registry.appendJobIdToCommitMessage (Just (JobId "job-123")) message `Assert.shouldEqual` "Update metadata for prelude\n\nJob ID: job-123" - -- This test exercises the Registry.handle to verify that readMetadata does + -- This test exercises Registry.handleRead to verify that readMetadata does -- not poison the AllMetadata cache: i.e. a single-package read must not seed -- the cache with a singleton map that readAllMetadata would mistake for the -- complete set. @@ -103,15 +103,15 @@ spec = do , { name: "control", version: "6.0.0", compilers: [ "0.15.15" ] } ] - -- | Run the REGISTRY effect - can't use the mock here because the regression + -- | Run the REGISTRY_READ effect - can't use the mock here because the regression -- | we are testing is in the caching code of the handle runRealRegistry :: forall a . RegistryEnv - -> Run (REGISTRY + GITHUB + LOG + EXCEPT String + AFF + EFFECT + ()) a + -> Run (REGISTRY_READ + GITHUB + LOG + EXCEPT String + AFF + EFFECT + ()) a -> Aff a runRealRegistry env = - Registry.interpret (Registry.handle env) + Registry.interpretRead (Registry.handleRead env) >>> GitHub.interpret handleGitHubStub >>> Log.interpret (\(Log _ _ next) -> pure next) >>> Except.catch (\err -> Run.liftAff (Aff.throwError (Aff.error err))) diff --git a/app/test/Test/Assert/Run.purs b/app/test/Test/Assert/Run.purs index 82aa1385..a427bd84 100644 --- a/app/test/Test/Assert/Run.purs +++ b/app/test/Test/Assert/Run.purs @@ -43,7 +43,7 @@ import Registry.App.Effect.PackageSets (PACKAGE_SETS, PackageSets(..)) import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Pursuit (PURSUIT, Pursuit(..)) import Registry.App.Effect.Pursuit as Pursuit -import Registry.App.Effect.Registry (REGISTRY, Registry(..)) +import Registry.App.Effect.Registry (REGISTRY, RegistryRead(..), RegistryWrite(..)) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (FetchError(..), SOURCE, Source(..)) import Registry.App.Effect.Source as Source @@ -134,8 +134,15 @@ runTestEffects env operation = Aff.attempt do githubCache <- liftEffect Cache.newCacheRef operation # Pursuit.interpret (handlePursuitMock { metadataRef: env.metadata, excludes: env.pursuitExcludes }) - # Registry.interpret - ( handleRegistryMock + # Registry.interpretWrite + ( handleRegistryWriteMock + { metadataRef: env.metadata + , indexRef: env.index + , failurePlan: env.failurePlan + } + ) + # Registry.interpretRead + ( handleRegistryReadMock { metadataRef: env.metadata , indexRef: env.index , failurePlan: env.failurePlan @@ -173,7 +180,8 @@ runRegistryMock :: forall a. Ref (Map PackageName Metadata) -> Ref ManifestIndex runRegistryMock metadataRef indexRef operation = do failurePlan <- liftEffect $ Ref.new [] operation - # Registry.interpret (handleRegistryMock { metadataRef, indexRef, failurePlan }) + # Registry.interpretWrite (handleRegistryWriteMock { metadataRef, indexRef, failurePlan }) + # Registry.interpretRead (handleRegistryReadMock { metadataRef, indexRef, failurePlan }) # runBaseEffects runGitHubCacheMemory :: forall r a. CacheRef -> Run (GITHUB_CACHE + LOG + EFFECT + r) a -> Run (LOG + EFFECT + r) a @@ -210,12 +218,34 @@ type RegistryMockEnv = , failurePlan :: Ref (Array TestFailure) } -handleRegistryMock :: forall r a. RegistryMockEnv -> Registry a -> Run (AFF + EFFECT + r) a -handleRegistryMock env = case _ of +handleRegistryReadMock :: forall r a. RegistryMockEnv -> RegistryRead a -> Run (AFF + EFFECT + r) a +handleRegistryReadMock env = case _ of ReadManifest name version reply -> do index <- Run.liftEffect (Ref.read env.indexRef) pure $ reply $ Right $ ManifestIndex.lookup name version index + ReadAllManifests reply -> do + index <- Run.liftEffect (Ref.read env.indexRef) + pure $ reply $ Right index + + ReadMetadata name reply -> do + metadata <- Run.liftEffect (Ref.read env.metadataRef) + pure $ reply $ Right $ Map.lookup name metadata + + ReadAllMetadata reply -> do + metadata <- Run.liftEffect (Ref.read env.metadataRef) + pure $ reply $ Right metadata + + -- FIXME: Actually reply with a package set + ReadLatestPackageSet reply -> + pure $ reply $ Right Nothing + + -- FIXME: Actually reply with a package set + ReadAllPackageSets reply -> + pure $ reply $ Right Map.empty + +handleRegistryWriteMock :: forall r a. RegistryMockEnv -> RegistryWrite a -> Run (AFF + EFFECT + r) a +handleRegistryWriteMock env = case _ of WriteManifest manifest reply -> do failWrite <- consumeFailure FailManifestWrite env.failurePlan if failWrite then do @@ -236,14 +266,6 @@ handleRegistryMock env = case _ of Run.liftEffect (Ref.write index' env.indexRef) pure $ reply $ Right unit - ReadAllManifests reply -> do - index <- Run.liftEffect (Ref.read env.indexRef) - pure $ reply $ Right index - - ReadMetadata name reply -> do - metadata <- Run.liftEffect (Ref.read env.metadataRef) - pure $ reply $ Right $ Map.lookup name metadata - WriteMetadata name metadata reply -> do failWrite <- consumeFailure FailMetadataWrite env.failurePlan if failWrite then do @@ -252,22 +274,10 @@ handleRegistryMock env = case _ of Run.liftEffect (Ref.modify_ (Map.insert name metadata) env.metadataRef) pure $ reply $ Right unit - ReadAllMetadata reply -> do - metadata <- Run.liftEffect (Ref.read env.metadataRef) - pure $ reply $ Right metadata - - -- FIXME: Actually reply with a package set - ReadLatestPackageSet reply -> - pure $ reply $ Right Nothing - -- FIXME: Actually write package set WritePackageSet _packageSet _message reply -> pure $ reply $ Right unit - -- FIXME: Actually reply with a package set - ReadAllPackageSets reply -> - pure $ reply $ Right Map.empty - -- FIXME: Actually mirror the package set MirrorPackageSet _packageSet reply -> pure $ reply $ Right unit diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index 52c75a1e..cf53e37e 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -40,7 +40,7 @@ import Registry.App.Effect.GitHub (GITHUB) import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.App.Legacy.LenientVersion as LenientVersion import Registry.Foreign.Octokit as Octokit @@ -100,7 +100,7 @@ main = launchAff_ do runDailyImport mode resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Verbose) # Env.runResourceEnv resourceEnv @@ -111,7 +111,7 @@ main = launchAff_ do liftEffect $ Process.exit' 1 Right _ -> pure unit -type DailyImportEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) +type DailyImportEffects = (REGISTRY_READ + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) runDailyImport :: Mode -> URL -> Run DailyImportEffects Unit runDailyImport mode registryApiUrl = do diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs index 35acc5cb..41026a7c 100644 --- a/scripts/src/PackageSetUpdater.purs +++ b/scripts/src/PackageSetUpdater.purs @@ -37,7 +37,7 @@ import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log import Registry.App.Effect.PackageSets as PackageSets -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.Foreign.Octokit as Octokit import Registry.Operation (PackageSetOperation(..)) @@ -98,7 +98,7 @@ main = launchAff_ do runPackageSetUpdater mode resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Normal) # Env.runResourceEnv resourceEnv @@ -109,7 +109,7 @@ main = launchAff_ do liftEffect $ Process.exit' 1 Right _ -> pure unit -type PackageSetUpdaterEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) +type PackageSetUpdaterEffects = (REGISTRY_READ + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) runPackageSetUpdater :: Mode -> URL -> Run PackageSetUpdaterEffects Unit runPackageSetUpdater mode registryApiUrl = do diff --git a/scripts/src/PackageTransferrer.purs b/scripts/src/PackageTransferrer.purs index dbfbf225..9e0e463c 100644 --- a/scripts/src/PackageTransferrer.purs +++ b/scripts/src/PackageTransferrer.purs @@ -37,7 +37,7 @@ import Registry.App.Effect.GitHub (GITHUB) import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry import Registry.Foreign.Octokit as Octokit import Registry.Location (Location(..)) @@ -103,7 +103,7 @@ main = launchAff_ do runPackageTransferrer mode maybePrivateKey resourceEnv.registryApiUrl # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) + # Registry.interpretRead (Registry.handleRead registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Normal) # Env.runResourceEnv resourceEnv @@ -114,7 +114,7 @@ main = launchAff_ do liftEffect $ Process.exit' 1 Right _ -> pure unit -type PackageTransferrerEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) +type PackageTransferrerEffects = (REGISTRY_READ + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) runPackageTransferrer :: Mode -> Maybe String -> URL -> Run PackageTransferrerEffects Unit runPackageTransferrer mode maybePrivateKey registryApiUrl = do diff --git a/scripts/src/VerifyIntegrity.purs b/scripts/src/VerifyIntegrity.purs index 414cb813..96c7fa5f 100644 --- a/scripts/src/VerifyIntegrity.purs +++ b/scripts/src/VerifyIntegrity.purs @@ -91,7 +91,7 @@ main = launchAff_ do let interpret = Except.catch (\error -> Run.liftEffect (Console.log error *> Process.exit' 1)) - >>> Registry.interpret (Registry.handle registryEnv) + >>> Registry.interpretRead (Registry.handleRead registryEnv) >>> GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) >>> Env.runResourceEnv resourceEnv >>> Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) @@ -127,7 +127,7 @@ main = launchAff_ do let interpret = Except.catch (\error -> Run.liftEffect (Console.log error *> Process.exit' 1)) - >>> Registry.interpret (Registry.handle registryEnv) + >>> Registry.interpretRead (Registry.handleRead registryEnv) >>> Storage.interpret (Storage.handleS3 { s3, cache }) >>> GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) >>> Env.runResourceEnv resourceEnv From 83fb1c30a1638ec4c13621f578851bfdc681aaa6 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Wed, 22 Jul 2026 20:27:58 +0000 Subject: [PATCH 2/8] Narrow registry read interpreter dependencies Amp-Thread-ID: https://ampcode.com/threads/T-019f8b54-076c-701a-863f-75106cd6f5bd Co-authored-by: Thomas Honeyman --- app-e2e/src/Test/E2E/Scripts.purs | 38 +-- app/src/App/Effect/Registry.purs | 395 ++++++++++++++--------------- app/test/App/Effect/Registry.purs | 10 +- scripts/src/PackageSetUpdater.purs | 14 +- scripts/src/VerifyIntegrity.purs | 13 +- 5 files changed, 221 insertions(+), 249 deletions(-) diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index 72a05661..424eec9a 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -149,8 +149,13 @@ spec = do Just _ -> Assert.fail "Expected PackageSetJob but got different job type" Nothing -> Assert.fail "Expected package set job to be enqueued" --- | Common environment for running registry scripts in E2E tests -type ScriptSetup = +-- | Common environment for running read-only registry scripts in E2E tests +type RegistryScriptSetup = + { resourceEnv :: Env.ResourceEnv + , registryEnv :: Registry.RegistryEnv + } + +type GitHubScriptSetup = { privateKey :: String , resourceEnv :: Env.ResourceEnv , registryEnv :: Registry.RegistryEnv @@ -159,17 +164,12 @@ type ScriptSetup = , githubCacheRef :: Cache.CacheRef } --- | Set up common environment for running registry scripts -setupScript :: E2E ScriptSetup -setupScript = do - { stateDir, privateKey } <- ask +setupRegistryScript :: E2E RegistryScriptSetup +setupRegistryScript = do + { stateDir } <- ask liftEffect $ Process.chdir stateDir resourceEnv <- liftEffect Env.lookupResourceEnv - token <- liftEffect $ Env.lookupRequired Env.githubToken - githubCacheRef <- liftAff Cache.newCacheRef registryCacheRef <- liftAff Cache.newCacheRef - let cache = Path.concat [ stateDir, "scratch", ".cache" ] - octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl debouncer <- liftAff Registry.newDebouncer let registryEnv :: Registry.RegistryEnv @@ -182,12 +182,22 @@ setupScript = do , debouncer , cacheRef: registryCacheRef } + pure { resourceEnv, registryEnv } + +setupGitHubScript :: E2E GitHubScriptSetup +setupGitHubScript = do + { stateDir, privateKey } <- ask + { resourceEnv, registryEnv } <- setupRegistryScript + token <- liftEffect $ Env.lookupRequired Env.githubToken + githubCacheRef <- liftAff Cache.newCacheRef + let cache = Path.concat [ stateDir, "scratch", ".cache" ] + octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl pure { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } -- | Run the DailyImporter script in Submit mode runDailyImporterScript :: E2E Unit runDailyImporterScript = do - { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript + { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupGitHubScript result <- liftAff $ DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl # Except.runExcept @@ -203,7 +213,7 @@ runDailyImporterScript = do -- | Run the PackageTransferrer script in Submit mode runPackageTransferrerScript :: E2E Unit runPackageTransferrerScript = do - { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript + { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupGitHubScript result <- liftAff $ PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl # Except.runExcept @@ -219,14 +229,12 @@ runPackageTransferrerScript = do -- | Run the PackageSetUpdater script in Submit mode runPackageSetUpdaterScript :: E2E Unit runPackageSetUpdaterScript = do - { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript + { resourceEnv, registryEnv } <- setupRegistryScript result <- liftAff $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit resourceEnv.registryApiUrl # Except.runExcept # Registry.interpretRead (Registry.handleRead registryEnv) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Quiet) - # Env.runResourceEnv resourceEnv # Run.runBaseAff' case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageSetUpdater failed: " <> err diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index 7ea05e47..4314990f 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -230,28 +230,86 @@ defaultRepos = , legacyPackageSets: Legacy.PackageSet.legacyPackageSetsRepo } -data RegistryOperation a - = ReadOperation (RegistryRead a) - | WriteOperation (RegistryWrite a) +-- | Get the upstream address associated with a repository key. +repoAddress :: RegistryEnv -> RepoKey -> Address +repoAddress env = case _ of + RegistryRepo -> env.repos.registry + ManifestIndexRepo -> env.repos.manifestIndex + LegacyPackageSetsRepo -> env.repos.legacyPackageSets + +-- | Get the local path for the checkout associated with a repository key. +repoPath :: RegistryEnv -> RepoKey -> FilePath +repoPath env = case _ of + RegistryRepo -> Path.concat [ env.workdir, "registry" ] + ManifestIndexRepo -> Path.concat [ env.workdir, "registry-index" ] + LegacyPackageSetsRepo -> Path.concat [ env.workdir, "package-sets" ] + +-- | Get the repository at the given key, recording whether the pull or clone +-- | had any effect (ie. if the repo was already up-to-date). +pull :: forall r. RegistryEnv -> RepoKey -> Run (LOG + AFF + EFFECT + r) (Either String GitResult) +pull env repoKey = do + let + path = repoPath env repoKey + address = repoAddress env repoKey + fetchLatest = do + Log.debug $ "Fetching repo at path " <> path + -- First, we need to verify whether we should clone or pull. + Run.liftAff (Aff.attempt (FS.Aff.stat path)) >>= case _ of + -- When the repository doesn't exist at the given file path we can go + -- straight to a clone. + Left _ -> do + let formatted = address.owner <> "/" <> address.repo + let url = "https://github.com/" <> formatted <> ".git" + Log.debug $ "Didn't find " <> formatted <> " locally, cloning..." + Run.liftAff (Git.gitCLI [ "clone", url, path ] Nothing) >>= case _ of + Left err -> do + Log.error $ "Failed to git clone repo " <> url <> " due to a git error: " <> err + pure $ Left $ "Could not read the repository at " <> formatted + Right _ -> + pure $ Right Git.Changed + + -- If it does, then we should pull. + Right _ -> do + Log.debug $ "Found repo at path " <> path <> ", pulling latest." + result <- Git.gitPull { address, pullMode: env.pull } path + pure result + + -- Check if the repo directory exists before consulting the debouncer. + -- This ensures that if the scratch directory is deleted (e.g., for test + -- isolation), we always re-clone rather than returning a stale NoChange. + repoExists <- Run.liftAff $ Aff.attempt (FS.Aff.stat path) + case repoExists of + Left _ -> do + -- Repo doesn't exist, bypass debouncer entirely and clone fresh + result <- fetchLatest + now <- nowUTC + Run.liftEffect $ Ref.modify_ (Map.insert path now) env.debouncer + pure result + Right _ -> do + -- Repo exists, check debouncer + now <- nowUTC + debouncers <- Run.liftEffect $ Ref.read env.debouncer + case Map.lookup path debouncers of + -- We will be behind the upstream by at most this amount of time. + Just prev | DateTime.diff now prev <= Duration.Minutes 1.0 -> + pure $ Right Git.NoChange + -- If we didn't debounce, then we should fetch the upstream. + _ -> do + result <- fetchLatest + Run.liftEffect $ Ref.modify_ (Map.insert path now) env.debouncer + pure result -- | Handle the REGISTRY_READ effect by downloading the registry and -- | registry-index repositories locally and reading their contents from disk. -handleRead :: forall r a. RegistryEnv -> RegistryRead a -> Run (GITHUB + LOG + AFF + EFFECT + r) a -handleRead env = handleOperation env <<< ReadOperation - --- | Handle the REGISTRY_WRITE effect by writing registry resources to disk. --- | Writes can optionally commit and push to the upstream Git repository. -handleWrite :: forall r a. RegistryEnv -> RegistryWrite a -> Run (GITHUB + LOG + AFF + EFFECT + r) a -handleWrite env = handleOperation env <<< WriteOperation - --- | These handlers share a memory-only cache: we do not want to cache on the --- | file system or other storage because they rely on the registry Git --- | repositories instead. -handleOperation :: forall r a. RegistryEnv -> RegistryOperation a -> Run (GITHUB + LOG + AFF + EFFECT + r) a -handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<< case _ of - ReadOperation (ReadManifest name version reply) -> do +-- | +-- | This handler uses the same memory-only cache as the write handler: we do +-- | not want to cache on the file system or other storage because the handlers +-- | rely on the registry Git repositories instead. +handleRead :: forall r a. RegistryEnv -> RegistryRead a -> Run (LOG + AFF + EFFECT + r) a +handleRead env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<< case _ of + ReadManifest name version reply -> do let formatted = formatPackageVersion name version - handleOperation env (ReadOperation (ReadAllManifests identity)) >>= case _ of + handleRead env (ReadAllManifests identity) >>= case _ of Left error -> pure $ reply $ Left error Right index -> case ManifestIndex.lookup name version index of Nothing -> do @@ -260,63 +318,15 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Just manifest -> do pure $ reply $ Right $ Just manifest - WriteOperation (WriteManifest manifest@(Manifest { name, version }) reply) -> map (map reply) Except.runExcept do - let formatted = formatPackageVersion name version - Log.info $ "Writing manifest for " <> formatted <> ":\n" <> printJson Manifest.codec manifest - index <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) - case ManifestIndex.insert ManifestIndex.ConsiderRanges manifest index of - Left error -> - Except.throw $ Array.fold - [ "Can't insert " <> formatted <> " into manifest index because it has unsatisfied dependencies:" - , printJson (Internal.Codec.packageMap Range.codec) error - ] - Right updated -> do - result <- writeCommitPush (CommitManifestEntry name) \indexPath -> do - ManifestIndex.insertIntoEntryFile indexPath manifest >>= case _ of - Left error -> Except.throw $ "Could not insert manifest for " <> formatted <> " into its entry file in WriteManifest: " <> error - Right _ -> pure $ Just $ "Update manifest for " <> formatted - case result of - Left error -> Except.throw $ "Failed to write and commit manifest: " <> error - Right r -> do - case r of - Git.NoChange -> Log.info "Did not commit manifest because it did not change." - Git.Changed -> Log.info "Wrote and committed manifest." - Cache.put _registryCache AllManifests updated - - WriteOperation (DeleteManifest name version reply) -> map (map reply) Except.runExcept do - let formatted = formatPackageVersion name version - Log.info $ "Deleting manifest for " <> formatted - index <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) - case ManifestIndex.delete ManifestIndex.ConsiderRanges name version index of - Left error -> - Except.throw $ Array.fold - [ "Can't delete " <> formatted <> " from manifest index because it would produce unsatisfied dependencies:" - , printJson (Internal.Codec.packageMap (Internal.Codec.versionMap (Internal.Codec.packageMap Range.codec))) error - ] - Right updated -> do - commitResult <- writeCommitPush (CommitManifestEntry name) \indexPath -> do - ManifestIndex.removeFromEntryFile indexPath name version >>= case _ of - Left error -> Except.throw $ "Could not remove manifest for " <> formatted <> " from its entry file in DeleteManifest: " <> error - Right _ -> pure $ Just $ "Remove manifest entry for " <> formatted - case commitResult of - Left error -> Except.throw $ "Failed to delete and commit manifest: " <> error - Right r -> do - case r of - Git.NoChange -> - Log.info "Did not commit manifest because it already didn't exist." - Git.Changed -> - Log.info "Wrote and committed manifest." - Cache.put _registryCache AllManifests updated - - ReadOperation (ReadAllManifests reply) -> map (map reply) Except.runExcept do + ReadAllManifests reply -> map (map reply) Except.runExcept do let refreshIndex = do - let indexPath = repoPath ManifestIndexRepo + let indexPath = repoPath env ManifestIndexRepo index <- readManifestIndexFromDisk indexPath Cache.put _registryCache AllManifests index pure index - pull ManifestIndexRepo >>= case _ of + pull env ManifestIndexRepo >>= case _ of Left error -> Except.throw $ "Could not read manifests because the manifest index repo could not be checked: " <> error Right Git.NoChange -> do @@ -330,9 +340,9 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.info "Manifest index has changed, replacing cache..." refreshIndex - ReadOperation (ReadMetadata name reply) -> map (map reply) Except.runExcept do + ReadMetadata name reply -> map (map reply) Except.runExcept do let printedName = PackageName.print name - let dir = repoPath RegistryRepo + let dir = repoPath env RegistryRepo let path = Path.concat [ dir, Constants.metadataDirectory, printedName <> ".json" ] @@ -362,7 +372,7 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.debug $ "Successfully read metadata for " <> printedName <> " from path " <> path pure (Just metadata) - pull RegistryRepo >>= case _ of + pull env RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read metadata because the registry repo could not be checked: " <> error @@ -390,38 +400,17 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Cache.delete _registryCache AllMetadata readMetadataFromDisk - WriteOperation (WriteMetadata name metadata reply) -> map (map reply) Except.runExcept do - let printedName = PackageName.print name - Log.info $ "Writing metadata for " <> printedName - Log.debug $ printJson Metadata.codec metadata - commitResult <- writeCommitPush (CommitMetadataEntry name) \dir -> do - let path = Path.concat [ dir, Constants.metadataDirectory, printedName <> ".json" ] - Run.liftAff (Aff.attempt (writeJsonFile Metadata.codec path metadata)) >>= case _ of - Left fsError -> Except.throw $ "Failed to write metadata for " <> printedName <> " to path " <> path <> " do to an fs error: " <> Aff.message fsError - Right _ -> pure $ Just $ "Update metadata for " <> printedName - case commitResult of - Left error -> Except.throw $ "Failed to write and commit metadata: " <> error - Right r -> do - case r of - Git.NoChange -> - Log.info "Did not commit metadata because it was unchanged." - Git.Changed -> - Log.info "Wrote and committed metadata." - cache <- Cache.get _registryCache AllMetadata - for_ cache \cached -> - Cache.put _registryCache AllMetadata (Map.insert name metadata cached) - - ReadOperation (ReadAllMetadata reply) -> map (map reply) Except.runExcept do + ReadAllMetadata reply -> map (map reply) Except.runExcept do let refreshMetadata = do - let dir = repoPath RegistryRepo + let dir = repoPath env RegistryRepo let metadataDir = Path.concat [ dir, Constants.metadataDirectory ] Log.info $ "Reading metadata for all packages from directory " <> metadataDir allMetadata <- readAllMetadataFromDisk metadataDir Cache.put _registryCache AllMetadata allMetadata pure allMetadata - pull RegistryRepo >>= case _ of + pull env RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read metadata because the registry repo could not be checked: " <> error Right Git.NoChange -> do @@ -435,11 +424,11 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.info "Registry repo has changed, replacing metadata cache..." refreshMetadata - ReadOperation (ReadLatestPackageSet reply) -> map (map reply) Except.runExcept do - pull RegistryRepo >>= case _ of + ReadLatestPackageSet reply -> map (map reply) Except.runExcept do + pull env RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error Right _ -> pure unit - let dir = repoPath RegistryRepo + let dir = repoPath env RegistryRepo let packageSetsDir = Path.concat [ dir, Constants.packageSetsDirectory ] Log.info $ "Reading latest package set from directory " <> packageSetsDir versions <- listPackageSetVersions packageSetsDir @@ -456,27 +445,11 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.debug $ "Successfully read package set " <> printed pure $ Just set - WriteOperation (WritePackageSet set@(PackageSet { version }) message reply) -> map (map reply) Except.runExcept do - pull RegistryRepo >>= case _ of + ReadAllPackageSets reply -> map (map reply) Except.runExcept do + pull env RegistryRepo >>= case _ of Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error Right _ -> pure unit - let name = Version.print version - Log.info $ "Writing package set " <> name - commitResult <- writeCommitPush (CommitPackageSet version) \dir -> do - let path = Path.concat [ dir, Constants.packageSetsDirectory, name <> ".json" ] - Run.liftAff (Aff.attempt (writeJsonFile PackageSet.codec path set)) >>= case _ of - Left fsError -> Except.throw $ "Failed to write package set " <> name <> " to path " <> path <> " do to an fs error: " <> Aff.message fsError - Right _ -> pure $ Just message - case commitResult of - Left error -> Except.throw $ "Failed to write and commit package set: " <> error - Right Git.NoChange -> Log.info "Did not commit package set because it was unchanged." - Right Git.Changed -> Log.info "Wrote and committed package set." - - ReadOperation (ReadAllPackageSets reply) -> map (map reply) Except.runExcept do - pull RegistryRepo >>= case _ of - Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error - Right _ -> pure unit - let dir = repoPath RegistryRepo + let dir = repoPath env RegistryRepo let packageSetsDir = Path.concat [ dir, Constants.packageSetsDirectory ] Log.info $ "Reading all package sets from directory " <> packageSetsDir versions <- listPackageSetVersions packageSetsDir @@ -494,13 +467,104 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.warn $ "Some package sets could not be read and were skipped: " <> Array.foldMap format xs pure $ Map.fromFoldable results.success +-- | Handle the REGISTRY_WRITE effect by writing registry resources to disk. +-- | Writes can optionally commit and push to the upstream Git repository. +-- | +-- | This handler uses the same memory-only cache as the read handler. +handleWrite :: forall r a. RegistryEnv -> RegistryWrite a -> Run (GITHUB + LOG + AFF + EFFECT + r) a +handleWrite env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) <<< case _ of + WriteManifest manifest@(Manifest { name, version }) reply -> map (map reply) Except.runExcept do + let formatted = formatPackageVersion name version + Log.info $ "Writing manifest for " <> formatted <> ":\n" <> printJson Manifest.codec manifest + index <- Except.rethrow =<< handleRead env (ReadAllManifests identity) + case ManifestIndex.insert ManifestIndex.ConsiderRanges manifest index of + Left error -> + Except.throw $ Array.fold + [ "Can't insert " <> formatted <> " into manifest index because it has unsatisfied dependencies:" + , printJson (Internal.Codec.packageMap Range.codec) error + ] + Right updated -> do + result <- writeCommitPush (CommitManifestEntry name) \indexPath -> do + ManifestIndex.insertIntoEntryFile indexPath manifest >>= case _ of + Left error -> Except.throw $ "Could not insert manifest for " <> formatted <> " into its entry file in WriteManifest: " <> error + Right _ -> pure $ Just $ "Update manifest for " <> formatted + case result of + Left error -> Except.throw $ "Failed to write and commit manifest: " <> error + Right r -> do + case r of + Git.NoChange -> Log.info "Did not commit manifest because it did not change." + Git.Changed -> Log.info "Wrote and committed manifest." + Cache.put _registryCache AllManifests updated + + DeleteManifest name version reply -> map (map reply) Except.runExcept do + let formatted = formatPackageVersion name version + Log.info $ "Deleting manifest for " <> formatted + index <- Except.rethrow =<< handleRead env (ReadAllManifests identity) + case ManifestIndex.delete ManifestIndex.ConsiderRanges name version index of + Left error -> + Except.throw $ Array.fold + [ "Can't delete " <> formatted <> " from manifest index because it would produce unsatisfied dependencies:" + , printJson (Internal.Codec.packageMap (Internal.Codec.versionMap (Internal.Codec.packageMap Range.codec))) error + ] + Right updated -> do + commitResult <- writeCommitPush (CommitManifestEntry name) \indexPath -> do + ManifestIndex.removeFromEntryFile indexPath name version >>= case _ of + Left error -> Except.throw $ "Could not remove manifest for " <> formatted <> " from its entry file in DeleteManifest: " <> error + Right _ -> pure $ Just $ "Remove manifest entry for " <> formatted + case commitResult of + Left error -> Except.throw $ "Failed to delete and commit manifest: " <> error + Right r -> do + case r of + Git.NoChange -> + Log.info "Did not commit manifest because it already didn't exist." + Git.Changed -> + Log.info "Wrote and committed manifest." + Cache.put _registryCache AllManifests updated + + WriteMetadata name metadata reply -> map (map reply) Except.runExcept do + let printedName = PackageName.print name + Log.info $ "Writing metadata for " <> printedName + Log.debug $ printJson Metadata.codec metadata + commitResult <- writeCommitPush (CommitMetadataEntry name) \dir -> do + let path = Path.concat [ dir, Constants.metadataDirectory, printedName <> ".json" ] + Run.liftAff (Aff.attempt (writeJsonFile Metadata.codec path metadata)) >>= case _ of + Left fsError -> Except.throw $ "Failed to write metadata for " <> printedName <> " to path " <> path <> " do to an fs error: " <> Aff.message fsError + Right _ -> pure $ Just $ "Update metadata for " <> printedName + case commitResult of + Left error -> Except.throw $ "Failed to write and commit metadata: " <> error + Right r -> do + case r of + Git.NoChange -> + Log.info "Did not commit metadata because it was unchanged." + Git.Changed -> + Log.info "Wrote and committed metadata." + cache <- Cache.get _registryCache AllMetadata + for_ cache \cached -> + Cache.put _registryCache AllMetadata (Map.insert name metadata cached) + + WritePackageSet set@(PackageSet { version }) message reply -> map (map reply) Except.runExcept do + pull env RegistryRepo >>= case _ of + Left error -> Except.throw $ "Could not read package sets because the registry repo could not be checked: " <> error + Right _ -> pure unit + let name = Version.print version + Log.info $ "Writing package set " <> name + commitResult <- writeCommitPush (CommitPackageSet version) \dir -> do + let path = Path.concat [ dir, Constants.packageSetsDirectory, name <> ".json" ] + Run.liftAff (Aff.attempt (writeJsonFile PackageSet.codec path set)) >>= case _ of + Left fsError -> Except.throw $ "Failed to write package set " <> name <> " to path " <> path <> " do to an fs error: " <> Aff.message fsError + Right _ -> pure $ Just message + case commitResult of + Left error -> Except.throw $ "Failed to write and commit package set: " <> error + Right Git.NoChange -> Log.info "Did not commit package set because it was unchanged." + Right Git.Changed -> Log.info "Wrote and committed package set." + -- https://github.com/purescript/package-sets/blob/psc-0.15.4-20220829/release.sh -- https://github.com/purescript/package-sets/blob/psc-0.15.4-20220829/update-latest-compatible-sets.sh - WriteOperation (MirrorPackageSet set@(PackageSet { version }) reply) -> map (map reply) Except.runExcept do + MirrorPackageSet set@(PackageSet { version }) reply -> map (map reply) Except.runExcept do let name = Version.print version Log.info $ "Mirroring legacy package set " <> name <> " to the legacy package sets repo" - manifests <- Except.rethrow =<< handleOperation env (ReadOperation (ReadAllManifests identity)) + manifests <- Except.rethrow =<< handleRead env (ReadAllManifests identity) Log.debug $ "Converting package set..." converted <- case Legacy.PackageSet.convertPackageSet manifests set of @@ -508,7 +572,7 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Right converted -> pure converted let printedTag = Legacy.PackageSet.printPscTag converted.tag - let legacyRepo = repoAddress LegacyPackageSetsRepo + let legacyRepo = repoAddress env LegacyPackageSetsRepo packageSetsTags <- GitHub.listTags legacyRepo >>= case _ of Left githubError -> @@ -599,20 +663,6 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Right Git.Changed -> Log.info "Pushed new tags to legacy registry." where - -- | Get the upstream address associated with a repository key - repoAddress :: RepoKey -> Address - repoAddress = case _ of - RegistryRepo -> env.repos.registry - ManifestIndexRepo -> env.repos.manifestIndex - LegacyPackageSetsRepo -> env.repos.legacyPackageSets - - -- | Get local filepath for the checkout associated with a repository key - repoPath :: RepoKey -> FilePath - repoPath = case _ of - RegistryRepo -> Path.concat [ env.workdir, "registry" ] - ManifestIndexRepo -> Path.concat [ env.workdir, "registry-index" ] - LegacyPackageSetsRepo -> Path.concat [ env.workdir, "package-sets" ] - -- | Write a file to the repository associated with the commit key, given a -- | callback that takes the file path of the repository on disk, writes the -- | file(s), and returns a commit message which is used to commit to the @@ -620,10 +670,10 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac writeCommitPush :: CommitKey -> (FilePath -> Run _ (Maybe String)) -> Run _ (Either String GitResult) writeCommitPush commitKey write = do let repoKey = commitKeyToRepoKey commitKey - pull repoKey >>= case _ of + pull env repoKey >>= case _ of Left error -> pure (Left error) Right _ -> do - let path = repoPath repoKey + let path = repoPath env repoKey write path >>= case _ of Nothing -> pure $ Left $ "Failed to write file(s) to " <> path Just message -> commit commitKey message >>= case _ of @@ -640,67 +690,12 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Nothing -> pure (Right Git.NoChange) Just { head } -> pure (Left head) - -- | Get the repository at the given key, recording whether the pull or clone - -- | had any effect (ie. if the repo was already up-to-date). - pull :: RepoKey -> Run _ (Either String GitResult) - pull repoKey = do - let - path = repoPath repoKey - address = repoAddress repoKey - fetchLatest = do - Log.debug $ "Fetching repo at path " <> path - -- First, we need to verify whether we should clone or pull. - Run.liftAff (Aff.attempt (FS.Aff.stat path)) >>= case _ of - -- When the repository doesn't exist at the given file path we can go - -- straight to a clone. - Left _ -> do - let formatted = address.owner <> "/" <> address.repo - let url = "https://github.com/" <> formatted <> ".git" - Log.debug $ "Didn't find " <> formatted <> " locally, cloning..." - Run.liftAff (Git.gitCLI [ "clone", url, path ] Nothing) >>= case _ of - Left err -> do - Log.error $ "Failed to git clone repo " <> url <> " due to a git error: " <> err - pure $ Left $ "Could not read the repository at " <> formatted - Right _ -> - pure $ Right Git.Changed - - -- If it does, then we should pull. - Right _ -> do - Log.debug $ "Found repo at path " <> path <> ", pulling latest." - result <- Git.gitPull { address, pullMode: env.pull } path - pure result - - -- Check if the repo directory exists before consulting the debouncer. - -- This ensures that if the scratch directory is deleted (e.g., for test - -- isolation), we always re-clone rather than returning a stale NoChange. - repoExists <- Run.liftAff $ Aff.attempt (FS.Aff.stat path) - case repoExists of - Left _ -> do - -- Repo doesn't exist, bypass debouncer entirely and clone fresh - result <- fetchLatest - now <- nowUTC - Run.liftEffect $ Ref.modify_ (Map.insert path now) env.debouncer - pure result - Right _ -> do - -- Repo exists, check debouncer - now <- nowUTC - debouncers <- Run.liftEffect $ Ref.read env.debouncer - case Map.lookup path debouncers of - -- We will be behind the upstream by at most this amount of time. - Just prev | DateTime.diff now prev <= Duration.Minutes 1.0 -> - pure $ Right Git.NoChange - -- If we didn't debounce, then we should fetch the upstream. - _ -> do - result <- fetchLatest - Run.liftEffect $ Ref.modify_ (Map.insert path now) env.debouncer - pure result - -- | Commit the file(s) indicated by the commit key with a commit message. commit :: CommitKey -> String -> Run _ (Either String GitResult) commit commitKey message' = do let repoKey = commitKeyToRepoKey commitKey - address = repoAddress repoKey + address = repoAddress env repoKey formatted = address.owner <> "/" <> address.repo message = appendJobIdToCommitMessage env.jobId message' case env.write of @@ -708,27 +703,27 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac Log.info $ "Skipping commit to repo " <> formatted <> " because write mode is 'ReadOnly'." pure $ Right Git.NoChange CommitAs committer -> do - result <- Git.gitCommit { committer, address, commit: commitKeyToPaths commitKey, message } (repoPath repoKey) + result <- Git.gitCommit { committer, address, commit: commitKeyToPaths commitKey, message } (repoPath env repoKey) pure result -- | Push the repository at the given key, recording whether the push had any -- | effect (ie. if the repo was already up-to-date). push :: RepoKey -> Run _ (Either String GitResult) push repoKey = do - let address = repoAddress repoKey + let address = repoAddress env repoKey let formatted = address.owner <> "/" <> address.repo case env.write of ReadOnly -> do Log.info $ "Skipping push to repo " <> formatted <> " because write mode is 'ReadOnly'." pure $ Right Git.NoChange CommitAs committer -> do - result <- Git.gitPush { address, committer } (repoPath repoKey) + result <- Git.gitPush { address, committer } (repoPath env repoKey) pure result -- | Tag the repository at the given key at its current commit with the tag tag :: RepoKey -> String -> Run _ (Either String GitResult) tag repoKey ref = do - let address = repoAddress repoKey + let address = repoAddress env repoKey let formatted = address.owner <> "/" <> address.repo case env.write of ReadOnly -> do @@ -736,7 +731,7 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac pure $ Right Git.NoChange CommitAs committer -> do result <- Except.runExcept do - let cwd = repoPath repoKey + let cwd = repoPath env repoKey existingTags <- Git.withGit cwd [ "tag", "--list" ] \error -> "Failed to list tags in local checkout " <> cwd <> ": " <> error if Array.elem ref $ String.split (String.Pattern "\n") existingTags then do @@ -755,7 +750,7 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac -- | Push the repository tags at the given key to its upstream. pushTags :: RepoKey -> Run _ (Either String GitResult) pushTags repoKey = do - let address = repoAddress repoKey + let address = repoAddress env repoKey let formatted = address.owner <> "/" <> address.repo case env.write of ReadOnly -> do @@ -763,7 +758,7 @@ handleOperation env = Cache.interpret _registryCache (Cache.handleMemory env.cac pure $ Right Git.NoChange CommitAs committer -> do result <- Except.runExcept do - let cwd = repoPath repoKey + let cwd = repoPath env repoKey let Git.AuthOrigin authOrigin = Git.mkAuthOrigin address committer output <- Git.withGit cwd [ "push", "--tags", authOrigin ] \error -> "Failed to push tags in local checkout " <> cwd <> ": " <> error diff --git a/app/test/App/Effect/Registry.purs b/app/test/App/Effect/Registry.purs index e9f2f7f0..268cf71a 100644 --- a/app/test/App/Effect/Registry.purs +++ b/app/test/App/Effect/Registry.purs @@ -9,8 +9,6 @@ import Node.Path as Path import Registry.API.V1 (JobId(..)) import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.GitHub (GITHUB, GitHub) -import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG, Log(..)) import Registry.App.Effect.Log as Log import Registry.App.Effect.Registry (REGISTRY_READ, RegistryEnv, WriteMode(..)) @@ -108,16 +106,10 @@ spec = do runRealRegistry :: forall a . RegistryEnv - -> Run (REGISTRY_READ + GITHUB + LOG + EXCEPT String + AFF + EFFECT + ()) a + -> Run (REGISTRY_READ + LOG + EXCEPT String + AFF + EFFECT + ()) a -> Aff a runRealRegistry env = Registry.interpretRead (Registry.handleRead env) - >>> GitHub.interpret handleGitHubStub >>> Log.interpret (\(Log _ _ next) -> pure next) >>> Except.catch (\err -> Run.liftAff (Aff.throwError (Aff.error err))) >>> Run.runBaseAff' - - -- | Stub GitHub handler — crashes if called. ReadMetadata and ReadAllMetadata - -- | don't use the GITHUB effect, so this should never be reached. - handleGitHubStub :: forall r a. GitHub a -> Run r a - handleGitHubStub _ = unsafeCrashWith "GITHUB effect should not be called in this test" diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs index 41026a7c..d1f83370 100644 --- a/scripts/src/PackageSetUpdater.purs +++ b/scripts/src/PackageSetUpdater.purs @@ -6,7 +6,6 @@ -- | nix run .#package-set-updater -- --submit # Actually submit to the API -- | -- | Required environment variables: --- | GITHUB_TOKEN - GitHub API token -- | REGISTRY_API_URL - Registry API URL (default: https://registry.purescript.org) module Registry.Scripts.PackageSetUpdater where @@ -25,21 +24,16 @@ import Effect.Class.Console as Console import Fetch (Method(..)) import Fetch as Fetch import JSON as JSON -import Node.Path as Path import Node.Process as Process import Registry.API.V1 as V1 import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.Env (RESOURCE_ENV) import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub (GITHUB) -import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry -import Registry.Foreign.Octokit as Octokit import Registry.Operation (PackageSetOperation(..)) import Registry.Operation as Operation import Registry.PackageName as PackageName @@ -75,13 +69,9 @@ main = launchAff_ do Env.loadEnvFile ".env" resourceEnv <- Env.lookupResourceEnv - token <- Env.lookupRequired Env.githubToken - githubCacheRef <- Cache.newCacheRef registryCacheRef <- Cache.newCacheRef - let cache = Path.concat [ scratchDir, ".cache" ] - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl debouncer <- Registry.newDebouncer let @@ -99,9 +89,7 @@ main = launchAff_ do runPackageSetUpdater mode resourceEnv.registryApiUrl # Except.runExcept # Registry.interpretRead (Registry.handleRead registryEnv) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) # Log.interpret (Log.handleTerminal Normal) - # Env.runResourceEnv resourceEnv # Run.runBaseAff' >>= case _ of Left err -> do @@ -109,7 +97,7 @@ main = launchAff_ do liftEffect $ Process.exit' 1 Right _ -> pure unit -type PackageSetUpdaterEffects = (REGISTRY_READ + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) +type PackageSetUpdaterEffects = (REGISTRY_READ + LOG + EXCEPT String + AFF + EFFECT + ()) runPackageSetUpdater :: Mode -> URL -> Run PackageSetUpdaterEffects Unit runPackageSetUpdater mode registryApiUrl = do diff --git a/scripts/src/VerifyIntegrity.purs b/scripts/src/VerifyIntegrity.purs index 96c7fa5f..86833325 100644 --- a/scripts/src/VerifyIntegrity.purs +++ b/scripts/src/VerifyIntegrity.purs @@ -24,13 +24,11 @@ import Node.Process as Process import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache as Cache import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub as GitHub import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Storage as Storage import Registry.Foreign.FSExtra as FS.Extra -import Registry.Foreign.Octokit as Octokit import Registry.Internal.Format as Internal.Format import Registry.ManifestIndex as ManifestIndex import Registry.PackageName as PackageName @@ -54,12 +52,10 @@ main = launchAff_ do -- Environment _ <- Env.loadEnvFile ".env" - resourceEnv <- Env.lookupResourceEnv -- Caching let cache = Path.concat [ scratchDir, ".cache" ] FS.Extra.ensureDirectory cache - githubCacheRef <- Cache.newCacheRef registryCacheRef <- Cache.newCacheRef -- Registry @@ -85,15 +81,10 @@ main = launchAff_ do log $ "Logs available at " <> logPath if localMode then do - token <- Env.lookupRequired Env.pacchettibottiToken - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl - let interpret = Except.catch (\error -> Run.liftEffect (Console.log error *> Process.exit' 1)) >>> Registry.interpretRead (Registry.handleRead registryEnv) - >>> GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - >>> Env.runResourceEnv resourceEnv >>> Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) >>> Run.runBaseAff' @@ -120,16 +111,14 @@ main = launchAff_ do liftEffect $ Process.exit' 1 else do - token <- Env.lookupRequired Env.pacchettibottiToken + resourceEnv <- Env.lookupResourceEnv s3 <- lift2 { key: _, secret: _ } (Env.lookupRequired Env.spacesKey) (Env.lookupRequired Env.spacesSecret) - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl let interpret = Except.catch (\error -> Run.liftEffect (Console.log error *> Process.exit' 1)) >>> Registry.interpretRead (Registry.handleRead registryEnv) >>> Storage.interpret (Storage.handleS3 { s3, cache }) - >>> GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) >>> Env.runResourceEnv resourceEnv >>> Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) >>> Run.runBaseAff' From 4a5dcc98b599802aa0f7f49fdeb4d2a49121e0ad Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Wed, 22 Jul 2026 20:57:28 +0000 Subject: [PATCH 3/8] Add registry version check script Amp-Thread-ID: https://ampcode.com/threads/T-019f8b91-2aa1-72c0-9046-7862234646fd Co-authored-by: Thomas Honeyman --- nix/overlay.nix | 4 ++ scripts/spago.yaml | 6 +++ scripts/src/PackageSetUpdater.purs | 64 +++++++++++++++---------- scripts/src/VersionCheck.purs | 76 ++++++++++++++++++++++++++++++ scripts/test/Main.purs | 75 +++++++++++++++++++++++++++++ spago.lock | 6 ++- 6 files changed, 206 insertions(+), 25 deletions(-) create mode 100644 scripts/src/VersionCheck.purs create mode 100644 scripts/test/Main.purs diff --git a/nix/overlay.nix b/nix/overlay.nix index 54fa965e..c344a578 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -62,6 +62,10 @@ let module = "Registry.Scripts.PackageTransferrer"; description = "Check for moved packages and submit transfer jobs"; }; + version-check = { + module = "Registry.Scripts.VersionCheck"; + description = "Report registry versions pending a package set update"; + }; verify-integrity = { module = "Registry.Scripts.VerifyIntegrity"; description = "Verify registry and registry-index consistency"; diff --git a/scripts/spago.yaml b/scripts/spago.yaml index a3bdb19f..76d33506 100644 --- a/scripts/spago.yaml +++ b/scripts/spago.yaml @@ -25,3 +25,9 @@ package: - registry-lib - run - strings + test: + main: Test.Registry.Scripts.Main + dependencies: + - registry-test-utils + - spec + - spec-node diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs index d1f83370..ab9571c7 100644 --- a/scripts/src/PackageSetUpdater.purs +++ b/scripts/src/PackageSetUpdater.purs @@ -48,6 +48,13 @@ data Mode = DryRun | Submit derive instance Eq Mode +-- | Which registry releases should be considered for a package set update. +-- | `AllPending` only reports upgrades to packages already in the set; adding +-- | packages that have never appeared in a package set is a separate concern. +data CandidateSelection + = RecentUploads DateTime.DateTime Hours + | AllPending + parser :: ArgParser Mode parser = Arg.choose "command" [ Arg.flag [ "dry-run" ] @@ -111,16 +118,9 @@ runPackageSetUpdater mode registryApiUrl = do Just set -> pure (Just set) for_ latestPackageSet \packageSet -> do - let currentPackages = (un PackageSet packageSet).packages - -- Find packages uploaded in the last 24 hours - recentUploads <- findRecentUploads (Hours 24.0) - let - -- Filter out packages already in the set at the same or newer version - newOrUpdated = recentUploads # Map.filterWithKey \name version -> - case Map.lookup name currentPackages of - Nothing -> true -- new package - Just currentVersion -> version > currentVersion -- upgrade + now <- nowUTC + newOrUpdated <- findPackageSetCandidates (RecentUploads now (Hours 24.0)) packageSet if Map.isEmpty newOrUpdated then Log.info "No new packages for package set update." @@ -175,23 +175,39 @@ runPackageSetUpdater mode registryApiUrl = do Right { jobId } -> do Log.info $ "Submitted package set job " <> unwrap jobId --- | Find the latest version of each package uploaded within the time limit -findRecentUploads :: Hours -> Run PackageSetUpdaterEffects (Map PackageName Version) -findRecentUploads limit = do +-- | Find the latest eligible registry version for each package relative to the +-- | current package set. +findPackageSetCandidates + :: forall r + . CandidateSelection + -> PackageSet + -> Run (REGISTRY_READ + EXCEPT String + r) (Map PackageName Version) +findPackageSetCandidates selection packageSet = do allMetadata <- Registry.readAllMetadata - now <- nowUTC - + pure $ selectPackageSetCandidates selection packageSet allMetadata + +-- | Select package set candidates in one traversal of registry metadata. +selectPackageSetCandidates + :: CandidateSelection + -> PackageSet + -> Map PackageName Metadata + -> Map PackageName Version +selectPackageSetCandidates selection (PackageSet packageSet) = Map.mapMaybeWithKey \name (Metadata metadata) -> do let - getLatestRecentVersion :: Metadata -> Maybe Version - getLatestRecentVersion (Metadata metadata) = do - let - recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) - \(Tuple version { publishedTime }) -> - if (DateTime.diff now publishedTime) <= limit then Just version else Nothing - Array.last $ Array.sort recentVersions - - pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> - map (Tuple name) $ getLatestRecentVersion metadata + eligibleVersions = Array.mapMaybe + ( \(Tuple version { publishedTime }) -> case selection of + RecentUploads now limit + | DateTime.diff now publishedTime <= limit -> Just version + RecentUploads _ _ -> Nothing + AllPending -> Just version + ) + (Map.toUnfoldable metadata.published) + latestVersion <- Array.last $ Array.sort eligibleVersions + case Map.lookup name packageSet.packages of + Just currentVersion + | latestVersion > currentVersion -> Just latestVersion + Nothing | RecentUploads _ _ <- selection -> Just latestVersion + _ -> Nothing -- | Submit a package set job to the registry API submitPackageSetJob :: String -> Operation.PackageSetUpdateRequest -> Aff (Either String V1.JobCreatedResponse) diff --git a/scripts/src/VersionCheck.purs b/scripts/src/VersionCheck.purs new file mode 100644 index 00000000..6f30be68 --- /dev/null +++ b/scripts/src/VersionCheck.purs @@ -0,0 +1,76 @@ +-- | Report the latest registry versions that are newer than their versions in +-- | the latest package set. +-- | +-- | Packages that have never appeared in a package set are intentionally not +-- | included; discovering those packages is tracked as separate follow-up work. +-- | +-- | Run via Nix: +-- | nix run .#version-check +module Registry.Scripts.VersionCheck where + +import Registry.App.Prelude + +import Data.Map as Map +import Effect.Class.Console as Console +import Node.Process as Process +import Registry.App.CLI.Git as Git +import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Log (LOG) +import Registry.App.Effect.Log as Log +import Registry.App.Effect.Registry (REGISTRY_READ) +import Registry.App.Effect.Registry as Registry +import Registry.PackageName as PackageName +import Registry.Scripts.PackageSetUpdater as PackageSetUpdater +import Registry.Version as Version +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except + +main :: Effect Unit +main = launchAff_ do + registryCacheRef <- Cache.newCacheRef + debouncer <- Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { jobId: Nothing + , pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + } + + result <- runVersionCheck + # Except.runExcept + # Registry.interpretRead (Registry.handleRead registryEnv) + # Log.interpret (Log.handleTerminal Normal) + # Run.runBaseAff' + + case result of + Left err -> do + Console.error $ "Error: " <> err + liftEffect $ Process.exit' 1 + Right _ -> pure unit + +type VersionCheckEffects = (REGISTRY_READ + LOG + EXCEPT String + AFF + EFFECT + ()) + +runVersionCheck :: Run VersionCheckEffects (Map PackageName Version) +runVersionCheck = do + Log.info "Registry Version Check: checking for pending package set updates..." + Registry.readLatestPackageSet >>= case _ of + Nothing -> do + Log.warn "No package set found, skipping version check." + pure Map.empty + Just packageSet -> do + pending <- PackageSetUpdater.findPackageSetCandidates PackageSetUpdater.AllPending packageSet + if Map.isEmpty pending then + Log.info "No package versions are pending a package set update." + else do + Log.info $ "Found " <> show (Map.size pending) <> " package versions pending a package set update:" + for_ (Map.toUnfoldable pending :: Array _) \(Tuple name version) -> + Log.info $ " - " <> PackageName.print name <> "@" <> Version.print version + pure pending diff --git a/scripts/test/Main.purs b/scripts/test/Main.purs new file mode 100644 index 00000000..9bacfbcb --- /dev/null +++ b/scripts/test/Main.purs @@ -0,0 +1,75 @@ +module Test.Registry.Scripts.Main (main) where + +import Registry.App.Prelude + +import Data.DateTime (DateTime) +import Data.Map as Map +import Data.Time.Duration (Hours(..)) +import Registry.Metadata (Metadata) +import Registry.Metadata as Metadata +import Registry.PackageName (PackageName) +import Registry.PackageSet (PackageSet) +import Registry.PackageSet as PackageSet +import Registry.Scripts.PackageSetUpdater as PackageSetUpdater +import Registry.Test.Assert as Assert +import Registry.Test.Utils as Utils +import Registry.Version (Version) +import Test.Spec as Spec +import Test.Spec.Reporter.Console (consoleReporter) +import Test.Spec.Runner.Node (runSpecAndExitProcess) + +main :: Effect Unit +main = runSpecAndExitProcess [ consoleReporter ] do + Spec.describe "PackageSetUpdater candidate selection" do + Spec.it "finds all newer versions of packages already in the package set" do + let + metadata = Map.fromFoldable + [ Utils.unsafeMetadata "prelude" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "effect" [ Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "older" [ Tuple "1.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "never-in-set" [ Tuple "1.0.0" [ "0.15.10" ] ] + ] + packageSet = mkPackageSet $ Map.fromFoldable + [ Tuple (Utils.unsafePackageName "prelude") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "effect") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "older") (Utils.unsafeVersion "2.0.0") + ] + expected = Map.singleton (Utils.unsafePackageName "prelude") (Utils.unsafeVersion "2.0.0") + + PackageSetUpdater.selectPackageSetCandidates PackageSetUpdater.AllPending packageSet metadata + `Assert.shouldEqual` expected + + Spec.it "preserves recent candidate selection for new packages and upgrades" do + let + oldTime = Utils.unsafeDateTime "2023-12-30T00:00:00.000Z" + now = Utils.unsafeDateTime "2024-01-02T00:00:00.000Z" + metadata = Map.fromFoldable + [ Utils.unsafeMetadata "prelude" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "effect" [ Tuple "1.0.0" [ "0.15.10" ] ] + , withPublishedTime oldTime $ Utils.unsafeMetadata "old-upload" [ Tuple "1.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "new-package" [ Tuple "1.0.0" [ "0.15.10" ] ] + ] + packageSet = mkPackageSet $ Map.fromFoldable + [ Tuple (Utils.unsafePackageName "prelude") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "effect") (Utils.unsafeVersion "1.0.0") + ] + expected = Map.fromFoldable + [ Tuple (Utils.unsafePackageName "prelude") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "new-package") (Utils.unsafeVersion "1.0.0") + ] + + PackageSetUpdater.selectPackageSetCandidates (PackageSetUpdater.RecentUploads now (Hours 24.0)) packageSet metadata + `Assert.shouldEqual` expected + +mkPackageSet :: Map PackageName Version -> PackageSet +mkPackageSet packages = PackageSet.PackageSet + { version: Utils.unsafeVersion "1.0.0" + , compiler: Utils.unsafeVersion "0.15.10" + , published: Utils.unsafeDate "2024-01-01" + , packages + } + +withPublishedTime :: forall a. DateTime -> Tuple a Metadata -> Tuple a Metadata +withPublishedTime publishedTime (Tuple name (Metadata.Metadata metadata)) = + Tuple name $ Metadata.Metadata $ metadata + { published = map (_ { publishedTime = publishedTime }) metadata.published } diff --git a/spago.lock b/spago.lock index d3b93a36..98749509 100644 --- a/spago.lock +++ b/spago.lock @@ -291,7 +291,11 @@ ] }, "test": { - "dependencies": [] + "dependencies": [ + "registry-test-utils", + "spec", + "spec-node" + ] } }, "registry-test-utils": { From 8acb229babe9d8da643776d7e90f48b6540225da Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Wed, 22 Jul 2026 21:15:50 +0000 Subject: [PATCH 4/8] Match script interpreter formatting Amp-Thread-ID: https://ampcode.com/threads/T-019f8b91-2aa1-72c0-9046-7862234646fd Co-authored-by: Thomas Honeyman --- scripts/src/VersionCheck.purs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/src/VersionCheck.purs b/scripts/src/VersionCheck.purs index 6f30be68..a89ccbfc 100644 --- a/scripts/src/VersionCheck.purs +++ b/scripts/src/VersionCheck.purs @@ -44,17 +44,16 @@ main = launchAff_ do , cacheRef: registryCacheRef } - result <- runVersionCheck + runVersionCheck # Except.runExcept # Registry.interpretRead (Registry.handleRead registryEnv) # Log.interpret (Log.handleTerminal Normal) # Run.runBaseAff' - - case result of - Left err -> do - Console.error $ "Error: " <> err - liftEffect $ Process.exit' 1 - Right _ -> pure unit + >>= case _ of + Left err -> do + Console.error $ "Error: " <> err + liftEffect $ Process.exit' 1 + Right _ -> pure unit type VersionCheckEffects = (REGISTRY_READ + LOG + EXCEPT String + AFF + EFFECT + ()) From d283822785f18302d0d3faf2a5c5ca80086ced80 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Mon, 27 Jul 2026 19:12:15 +0000 Subject: [PATCH 5/8] Plan and report package set updates --- .github/workflows/version-check.yml | 62 +++ scripts/src/VersionCheck.purs | 560 ++++++++++++++++++++++++++-- scripts/test/Main.purs | 128 +++++++ 3 files changed, 729 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/version-check.yml diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml new file mode 100644 index 00000000..e1a9b5f1 --- /dev/null +++ b/.github/workflows/version-check.yml @@ -0,0 +1,62 @@ +name: Weekly package set version check + +on: + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: weekly-package-set-version-check + cancel-in-progress: false + +jobs: + version-check: + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install Nix + uses: DeterminateSystems/determinate-nix-action@v3 + + - name: Setup Nix cache + uses: DeterminateSystems/magic-nix-cache-action@v14 + with: + use-flakehub: false + + - name: Plan and verify pending package set updates + run: nix run .#version-check -- --verify --output version-check.json --markdown version-check.md + + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: package-set-version-check + path: | + version-check.json + version-check.md + + - name: Update rolling report issue + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + + title='Weekly package set version report' + issue_number=$( + gh issue list --repo "$REPOSITORY" --state open --search "$title in:title" --limit 100 --json number,title \ + | jq -r --arg title "$title" '.[] | select(.title == $title) | .number' \ + | sed -n '1p' + ) + + if [[ -n "$issue_number" ]]; then + gh issue edit "$issue_number" --repo "$REPOSITORY" --body-file version-check.md + else + gh issue create --repo "$REPOSITORY" --title "$title" --body-file version-check.md + fi diff --git a/scripts/src/VersionCheck.purs b/scripts/src/VersionCheck.purs index a89ccbfc..eec62d62 100644 --- a/scripts/src/VersionCheck.purs +++ b/scripts/src/VersionCheck.purs @@ -1,34 +1,104 @@ --- | Report the latest registry versions that are newer than their versions in --- | the latest package set. +-- | Report registry versions newer than those in the latest package set and +-- | plan a compatible set of upgrades. -- | -- | Packages that have never appeared in a package set are intentionally not -- | included; discovering those packages is tracked as separate follow-up work. +-- | Planning is conservative: a release is eligible only after its metadata +-- | records a successful build with the package set compiler. +-- | +-- | Each pending package is a variable whose choices are its current version +-- | and all eligible newer versions. A bounded branch-and-bound search visits +-- | newer choices first and maximizes, in order, the number of upgraded packages +-- | and their cumulative freshness. Partial dependency-range checks prune +-- | impossible branches. Existing range mismatches are tolerated only when both +-- | ends of the dependency edge remain unchanged because historical package sets +-- | can contain stale declared bounds despite having compiled successfully. +-- | +-- | The search has a fixed node budget so the weekly report cannot hang on a +-- | pathological candidate graph, and any truncation is explicit in the report. +-- | The optional verification atomically compiles the exact planned payload but +-- | never publishes it. -- | -- | Run via Nix: -- | nix run .#version-check +-- | nix run .#version-check -- --verify --output report.json --markdown report.md module Registry.Scripts.VersionCheck where import Registry.App.Prelude +import ArgParse.Basic (ArgParser) +import ArgParse.Basic as Arg +import Data.Array as Array +import Data.Codec.JSON as CJ +import Data.Codec.JSON.Common as CJ.Common +import Data.Codec.JSON.Record as CJ.Record +import Data.DateTime (DateTime) +import Data.DateTime as DateTime +import Data.Foldable as Foldable +import Data.Int as Int import Data.Map as Map +import Data.Set as Set +import Data.String as String +import Data.Time.Duration (Days(..)) import Effect.Class.Console as Console +import Node.FS.Aff as FS.Aff +import Node.Path as Path import Node.Process as Process import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Env (RESOURCE_ENV) +import Registry.App.Effect.Env as Env import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageSets (PACKAGE_SETS) +import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry +import Registry.App.Effect.Storage (STORAGE) +import Registry.App.Effect.Storage as Storage +import Registry.Foreign.FSExtra as FS.Extra +import Registry.Internal.Codec as Internal.Codec +import Registry.Manifest (Manifest(..)) +import Registry.ManifestIndex (ManifestIndex) +import Registry.ManifestIndex as ManifestIndex +import Registry.Metadata (Metadata(..)) +import Registry.PackageName (PackageName) import Registry.PackageName as PackageName -import Registry.Scripts.PackageSetUpdater as PackageSetUpdater +import Registry.PackageSet (PackageSet(..)) +import Registry.Range (Range) +import Registry.Range as Range +import Registry.Version (Version) import Registry.Version as Version import Run (AFF, EFFECT, Run) import Run as Run import Run.Except (EXCEPT) import Run.Except as Except +type Options = + { markdown :: Maybe FilePath + , output :: Maybe FilePath + , verify :: Boolean + } + +parser :: ArgParser Options +parser = ado + verify <- Arg.flag [ "--verify" ] "Atomically compile the planned update without publishing it." # Arg.boolean # Arg.default false + output <- Arg.argument [ "--output" ] "Write the structured report as JSON." # Arg.optional + markdown <- Arg.argument [ "--markdown" ] "Write the human-readable report as Markdown." # Arg.optional + in { markdown, output, verify } + main :: Effect Unit main = launchAff_ do + args <- Array.drop 2 <$> liftEffect Process.argv + let description = "Report, plan, and optionally verify registry versions pending a package set update." + options <- case Arg.parseArgs "version-check" description parser args of + Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) + Right parsed -> pure parsed + + resourceEnv <- Env.lookupResourceEnv + + let cache = Path.concat [ scratchDir, ".cache" ] + FS.Extra.ensureDirectory cache registryCacheRef <- Cache.newCacheRef debouncer <- Registry.newDebouncer @@ -44,32 +114,480 @@ main = launchAff_ do , cacheRef: registryCacheRef } - runVersionCheck + runVersionCheck options.verify + # PackageSets.interpret (PackageSets.handle { workdir: scratchDir }) # Except.runExcept # Registry.interpretRead (Registry.handleRead registryEnv) + # Storage.interpret (Storage.handleReadOnly cache) + # Env.runResourceEnv resourceEnv # Log.interpret (Log.handleTerminal Normal) # Run.runBaseAff' >>= case _ of Left err -> do Console.error $ "Error: " <> err liftEffect $ Process.exit' 1 - Right _ -> pure unit + Right report -> do + let markdown = renderMarkdown report + Console.log markdown + for_ options.output \path -> writeJsonFile reportCodec path report + for_ options.markdown \path -> FS.Aff.writeTextFile UTF8 path (markdown <> "\n") + +type Incompatibility = + { dependency :: PackageName + , package :: PackageName + , packageDirectDependants :: Int + , packageTransitiveDependants :: Int + , range :: Range + , selectedVersion :: Maybe Version + , version :: Version + } + +type PendingPackage = + { blockedBy :: Array Incompatibility + , blockers :: Array Incompatibility + , candidate :: Version + , candidateCompilerCompatible :: Boolean + , candidatePublished :: DateTime + , current :: Version + , directDependants :: Int + , name :: PackageName + , planned :: Maybe Version + , staleDays :: Int + , staleSince :: DateTime + , transitiveDependants :: Int + } + +type Plan = + { error :: Maybe String + , packages :: Map PackageName Version + , searchTruncated :: Boolean + } + +type Verification = + { error :: Maybe String + , status :: String + } + +type Report = + { compiler :: Version + , generatedAt :: DateTime + , latestIncompatibilities :: Array Incompatibility + , packageSet :: Version + , pending :: Array PendingPackage + , plan :: Plan + , verification :: Verification + } + +incompatibilityCodec :: CJ.Codec Incompatibility +incompatibilityCodec = CJ.named "VersionCheckIncompatibility" $ CJ.Record.object + { dependency: PackageName.codec + , package: PackageName.codec + , packageDirectDependants: CJ.int + , packageTransitiveDependants: CJ.int + , range: Range.codec + , selectedVersion: CJ.Common.nullable Version.codec + , version: Version.codec + } -type VersionCheckEffects = (REGISTRY_READ + LOG + EXCEPT String + AFF + EFFECT + ()) +pendingPackageCodec :: CJ.Codec PendingPackage +pendingPackageCodec = CJ.named "VersionCheckPendingPackage" $ CJ.Record.object + { blockedBy: CJ.array incompatibilityCodec + , blockers: CJ.array incompatibilityCodec + , candidate: Version.codec + , candidateCompilerCompatible: CJ.boolean + , candidatePublished: Internal.Codec.iso8601DateTime + , current: Version.codec + , directDependants: CJ.int + , name: PackageName.codec + , planned: CJ.Common.nullable Version.codec + , staleDays: CJ.int + , staleSince: Internal.Codec.iso8601DateTime + , transitiveDependants: CJ.int + } -runVersionCheck :: Run VersionCheckEffects (Map PackageName Version) -runVersionCheck = do +planCodec :: CJ.Codec Plan +planCodec = CJ.named "VersionCheckPlan" $ CJ.Record.object + { error: CJ.Common.nullable CJ.string + , packages: Internal.Codec.packageMap Version.codec + , searchTruncated: CJ.boolean + } + +verificationCodec :: CJ.Codec Verification +verificationCodec = CJ.named "VersionCheckVerification" $ CJ.Record.object + { error: CJ.Common.nullable CJ.string + , status: CJ.string + } + +reportCodec :: CJ.Codec Report +reportCodec = CJ.named "VersionCheckReport" $ CJ.Record.object + { compiler: Version.codec + , generatedAt: Internal.Codec.iso8601DateTime + , latestIncompatibilities: CJ.array incompatibilityCodec + , packageSet: Version.codec + , pending: CJ.array pendingPackageCodec + , plan: planCodec + , verification: verificationCodec + } + +type Candidate = + { candidate :: Version + , candidatePublished :: DateTime + , compatibleVersions :: Array Version + , current :: Version + , name :: PackageName + , staleSince :: DateTime + } + +type Choice = + { freshness :: Int + , upgrade :: Int + , version :: Version + } + +type PlannerVariable = + { choices :: Array Choice + , name :: PackageName + } + +type PlanScore = + { freshness :: Int + , upgrades :: Int + } + +type ScoredPlan = + { packages :: Map PackageName Version + , score :: PlanScore + } + +type SearchState = + { best :: Maybe ScoredPlan + , remainingNodes :: Int + , truncated :: Boolean + } + +type VersionCheckEffects = (REGISTRY_READ + PACKAGE_SETS + STORAGE + RESOURCE_ENV + LOG + EXCEPT String + AFF + EFFECT + ()) + +runVersionCheck :: Boolean -> Run VersionCheckEffects Report +runVersionCheck verify = do Log.info "Registry Version Check: checking for pending package set updates..." - Registry.readLatestPackageSet >>= case _ of - Nothing -> do - Log.warn "No package set found, skipping version check." - pure Map.empty - Just packageSet -> do - pending <- PackageSetUpdater.findPackageSetCandidates PackageSetUpdater.AllPending packageSet - if Map.isEmpty pending then - Log.info "No package versions are pending a package set update." - else do - Log.info $ "Found " <> show (Map.size pending) <> " package versions pending a package set update:" - for_ (Map.toUnfoldable pending :: Array _) \(Tuple name version) -> - Log.info $ " - " <> PackageName.print name <> "@" <> Version.print version - pure pending + packageSet <- Registry.readLatestPackageSet >>= case _ of + Nothing -> Except.throw "No package set found." + Just found -> pure found + metadata <- Registry.readAllMetadata + manifests <- Registry.readAllManifests + generatedAt <- nowUTC + let initial = buildReport generatedAt packageSet metadata manifests + verification <- verifyPlan verify packageSet initial.plan + let report = initial { verification = verification } + Log.info $ "Found " <> show (Array.length report.pending) <> " package versions pending a package set update." + pure report + +verifyPlan + :: forall r + . Boolean + -> PackageSet + -> Plan + -> Run (PACKAGE_SETS + LOG + EXCEPT String + AFF + EFFECT + r) Verification +verifyPlan verify packageSet plan + | not verify = pure { error: Nothing, status: "not-run" } + | Just error <- plan.error = pure { error: Just error, status: "failed" } + | Map.isEmpty plan.packages = pure { error: Nothing, status: "not-needed" } + | otherwise = do + Log.info $ "Atomically compiling " <> show (Map.size plan.packages) <> " planned package updates..." + Except.runExcept (PackageSets.upgradeAtomic packageSet (un PackageSet packageSet).compiler (map PackageSets.Update plan.packages)) >>= case _ of + Left error -> pure { error: Just error, status: "failed" } + Right (Left error) -> pure { error: Just error, status: "failed" } + Right (Right _) -> pure { error: Nothing, status: "verified" } + +buildReport :: DateTime -> PackageSet -> Map PackageName Metadata -> ManifestIndex -> Report +buildReport generatedAt packageSet@(PackageSet set) metadata manifests = do + let candidates = findCandidates set.compiler set.packages metadata manifests + let latest = Map.fromFoldable $ candidates <#> \candidate -> Tuple candidate.name candidate.candidate + let proposed = Map.union latest set.packages + let + plan = case planPackageSet packageSet metadata manifests of + Left error -> { error: Just error, packages: Map.empty, searchTruncated: false } + Right { packages, searchTruncated } -> { error: Nothing, packages, searchTruncated } + let planned = Map.union plan.packages set.packages + let latestIncompatibilities = findIncompatibilities set.packages proposed manifests + let + pending = candidates <#> \candidate -> do + let direct = directDependants set.packages manifests candidate.name + let transitive = transitiveDependants set.packages manifests candidate.name + let candidateIncompatibilities = findIncompatibilities set.packages (Map.insert candidate.name candidate.candidate planned) manifests + { blockedBy: Array.filter (_.package >>> eq candidate.name) candidateIncompatibilities + , blockers: Array.filter (_.dependency >>> eq candidate.name) candidateIncompatibilities + , candidate: candidate.candidate + , candidateCompilerCompatible: Array.elem candidate.candidate candidate.compatibleVersions + , candidatePublished: candidate.candidatePublished + , current: candidate.current + , directDependants: Array.length direct + , name: candidate.name + , planned: Map.lookup candidate.name plan.packages + , staleDays: daysBetween candidate.staleSince generatedAt + , staleSince: candidate.staleSince + , transitiveDependants: Array.length transitive + } + { compiler: set.compiler + , generatedAt + , latestIncompatibilities + , packageSet: set.version + , pending + , plan + , verification: { error: Nothing, status: "not-run" } + } + +-- Candidate versions are restricted to releases known to compile with the +-- package set compiler and present in the manifest index. The report still +-- retains the actual latest release so incompatible latest releases stay +-- visible even when the planner chooses an intermediate version. +findCandidates + :: Version + -> Map PackageName Version + -> Map PackageName Metadata + -> ManifestIndex + -> Array Candidate +findCandidates compiler packages metadata manifests = Array.mapMaybe toCandidate (Map.toUnfoldable packages) + where + toCandidate (Tuple name current) = do + Metadata packageMetadata <- Map.lookup name metadata + let newer = Array.filter (fst >>> (_ > current)) (Map.toUnfoldable packageMetadata.published) + latest@(Tuple candidate candidateMetadata) <- Array.last $ Array.sortBy (comparing fst) newer + let Tuple _ staleMetadata = Foldable.minimumBy (comparing (_.publishedTime <<< snd)) newer # fromMaybe latest + let staleSince = staleMetadata.publishedTime + let + compatibleVersions = newer + # Array.mapMaybe + ( \(Tuple version published) -> do + guard $ Foldable.elem compiler published.compilers + _ <- ManifestIndex.lookup name version manifests + pure version + ) + # Array.sort + pure + { candidate + , candidatePublished: candidateMetadata.publishedTime + , compatibleVersions + , current + , name + , staleSince + } + +-- Maximize the number of upgraded packages, then their cumulative freshness, +-- while satisfying the exact selected dependency versions. Existing range +-- mismatches are tolerated only when both ends of that dependency edge remain +-- unchanged; this is necessary because package sets prove compatibility by +-- compilation and some historical manifests have stale bounds. +planPackageSet + :: PackageSet + -> Map PackageName Metadata + -> ManifestIndex + -> Either String { packages :: Map PackageName Version, searchTruncated :: Boolean } +planPackageSet (PackageSet set) metadata manifests = do + let candidates = findCandidates set.compiler set.packages metadata manifests + let variables = Array.mapMaybe toVariable candidates + let fixed = Foldable.foldl (flip Map.delete) set.packages (map _.name variables) + unless (completePlanIsValid manifests set.packages set.packages) do + Left "The current package set is missing a manifest or one of its declared dependencies." + let search = searchPlan manifests set.packages variables fixed + result <- note "No compatible package set update could be planned without introducing new dependency-range violations." search.best + pure + { packages: Map.filterWithKey + (\name version -> maybe false (version > _) (Map.lookup name set.packages)) + result.packages + , searchTruncated: search.truncated + } + where + toVariable candidate = do + guard $ not (Array.null candidate.compatibleVersions) + let + choices = + [ { freshness: 0, upgrade: 0, version: candidate.current } ] + <> Array.mapWithIndex + (\index version -> { freshness: index + 1, upgrade: 1, version }) + candidate.compatibleVersions + pure { choices, name: candidate.name } + +-- Branch-and-bound visits newest choices first and prunes a branch when its +-- best possible score cannot improve on the best complete plan already found. +-- The node budget keeps a pathological dependency graph from hanging the +-- weekly report; a truncated plan is reported but not automatically submitted. +searchPlan + :: ManifestIndex + -> Map PackageName Version + -> Array PlannerVariable + -> Map PackageName Version + -> { best :: Maybe ScoredPlan, truncated :: Boolean } +searchPlan manifests baseline variables fixed = do + let baselinePlan = { packages: baseline, score: { freshness: 0, upgrades: 0 } } + let result = go 0 fixed { freshness: 0, upgrades: 0 } { best: Just baselinePlan, remainingNodes: 50_000, truncated: false } + { best: result.best, truncated: result.truncated } + where + go :: Int -> Map PackageName Version -> PlanScore -> SearchState -> SearchState + go offset selected score state + | state.remainingNodes <= 0 = state { truncated = true } + | otherwise = do + let nextState = state { remainingNodes = state.remainingNodes - 1 } + let remaining = Array.drop offset variables + let potential = addScores score (maximumRemainingScore remaining) + if maybe false (not <<< betterScore potential <<< _.score) nextState.best then + nextState + else case Array.index variables offset of + Nothing -> + if completePlanIsValid manifests baseline selected then nextState { best = chooseBetter { packages: selected, score } nextState.best } else nextState + Just variable -> + Array.foldl + ( \currentState choice -> do + let nextSelected = Map.insert variable.name choice.version selected + let nextRemaining = Array.drop (offset + 1) variables + let nextScore = addScores score { freshness: choice.freshness, upgrades: choice.upgrade } + if partialPlanIsPossible manifests baseline nextSelected nextRemaining then + go (offset + 1) nextSelected nextScore currentState + else + currentState + ) + nextState + (Array.reverse variable.choices) + +maximumRemainingScore :: Array PlannerVariable -> PlanScore +maximumRemainingScore = Array.foldl addMaximum { freshness: 0, upgrades: 0 } + where + addMaximum score variable = case Array.last variable.choices of + Nothing -> score + Just choice -> addScores score { freshness: choice.freshness, upgrades: choice.upgrade } + +addScores :: PlanScore -> PlanScore -> PlanScore +addScores a b = + { freshness: a.freshness + b.freshness + , upgrades: a.upgrades + b.upgrades + } + +betterScore :: PlanScore -> PlanScore -> Boolean +betterScore a b = a.upgrades > b.upgrades || (a.upgrades == b.upgrades && a.freshness > b.freshness) + +chooseBetter :: ScoredPlan -> Maybe ScoredPlan -> Maybe ScoredPlan +chooseBetter candidate = case _ of + Nothing -> Just candidate + Just current | betterScore candidate.score current.score -> Just candidate + current -> current + +-- Check selected manifests against exact selected dependency versions. For an +-- unselected dependency, at least one of its remaining choices must satisfy +-- the range (or preserve an unchanged baseline mismatch). +partialPlanIsPossible :: ManifestIndex -> Map PackageName Version -> Map PackageName Version -> Array PlannerVariable -> Boolean +partialPlanIsPossible manifests baseline selected remaining = all manifestIsPossible (Map.toUnfoldable selected :: Array (Tuple PackageName Version)) + where + remainingChoices = Map.fromFoldable $ remaining <#> \variable -> Tuple variable.name variable.choices + + manifestIsPossible (Tuple name version) = fromMaybe false do + Manifest manifest <- ManifestIndex.lookup name version manifests + pure $ all (dependencyIsPossible name version) (Map.toUnfoldable manifest.dependencies :: Array (Tuple PackageName Range)) + + dependencyIsPossible name packageVersion (Tuple dependency range) = case Map.lookup dependency selected of + Just dependencyVersion -> choiceIsCompatible name packageVersion dependency range dependencyVersion + Nothing -> fromMaybe false do + choices <- Map.lookup dependency remainingChoices + pure $ any (choiceIsCompatible name packageVersion dependency range <<< _.version) choices + + choiceIsCompatible name packageVersion dependency range dependencyVersion = + Range.includes range dependencyVersion + || + ( Map.lookup name baseline == Just packageVersion + && Map.lookup dependency baseline + == Just dependencyVersion + ) + +completePlanIsValid :: ManifestIndex -> Map PackageName Version -> Map PackageName Version -> Boolean +completePlanIsValid manifests baseline selected = partialPlanIsPossible manifests baseline selected [] + +findIncompatibilities :: Map PackageName Version -> Map PackageName Version -> ManifestIndex -> Array Incompatibility +findIncompatibilities baseline selected manifests = do + Tuple package version <- Map.toUnfoldable selected + Manifest manifest <- Array.fromFoldable $ ManifestIndex.lookup package version manifests + Tuple dependency range <- Map.toUnfoldable manifest.dependencies + let selectedVersion = Map.lookup dependency selected + guard $ maybe true (not <<< Range.includes range) selectedVersion + pure + { dependency + , package + , packageDirectDependants: Array.length (directDependants baseline manifests package) + , packageTransitiveDependants: Array.length (transitiveDependants baseline manifests package) + , range + , selectedVersion + , version + } + +directDependants :: Map PackageName Version -> ManifestIndex -> PackageName -> Array PackageName +directDependants packages manifests dependency = Array.mapMaybe isDependant (Map.toUnfoldable packages) + where + isDependant (Tuple name version) = do + Manifest manifest <- ManifestIndex.lookup name version manifests + name <$ guard (Map.member dependency manifest.dependencies) + +transitiveDependants :: Map PackageName Version -> ManifestIndex -> PackageName -> Array PackageName +transitiveDependants packages manifests dependency = Array.fromFoldable $ Set.delete dependency $ go (Set.singleton dependency) [ dependency ] + where + go seen [] = seen + go seen frontier = do + let discovered = Set.fromFoldable (frontier >>= directDependants packages manifests) + let next = Set.difference discovered seen + go (Set.union seen next) (Array.fromFoldable next) + +daysBetween :: DateTime -> DateTime -> Int +daysBetween from to = do + let Days days = DateTime.diff to from + max 0 (Int.floor days) + +renderMarkdown :: Report -> String +renderMarkdown report = String.joinWith "\n" + [ "# Weekly package set version report" + , "" + , "Package set: `" <> Version.print report.packageSet <> "` (PureScript `" <> Version.print report.compiler <> "`)" + , "Generated: `" <> Internal.Codec.formatIso8601 report.generatedAt <> "`" + , "Verification: **" <> report.verification.status <> "**" + , maybe "" (\error -> "\nVerification error:\n```text\n" <> String.take 4_000 error <> "\n```") report.verification.error + , "" + , "## Planned update" + , "" + , if Map.isEmpty report.plan.packages then "No package updates were planned." + else String.joinWith "\n" do + Tuple name version <- Map.toUnfoldable report.plan.packages + [ "- `" <> PackageName.print name <> "@" <> Version.print version <> "`" ] + , if report.plan.searchTruncated then "\n⚠️ Planner search limit reached; this is the best compatible plan found so far and will not be submitted automatically." else "" + , maybe "" (\error -> "\nPlanner error: " <> error) report.plan.error + , "" + , "## Pending versions" + , "" + , renderPendingPackages report.pending + ] + where + renderPendingPackages pending + | Array.null pending = "No package versions are pending." + | otherwise = do + let visible = Array.take 100 pending + let omitted = Array.length pending - Array.length visible + String.joinWith "\n\n" (map renderPending visible) + <> if omitted > 0 then "\n\n_" <> show omitted <> " more pending packages are available in the JSON artifact._" else "" + + renderPending pending = String.joinWith "\n" + [ "### `" <> PackageName.print pending.name <> "` " <> Version.print pending.current <> " → " <> Version.print pending.candidate + , "" + , "- Stale for **" <> show pending.staleDays <> " days** (since `" <> Internal.Codec.formatIso8601 pending.staleSince <> "`)" + , "- Dependants in the current set: **" <> show pending.directDependants <> " direct**, **" <> show pending.transitiveDependants <> " transitive**" + , "- Planned version: " <> maybe "none" (\version -> "`" <> Version.print version <> "`") pending.planned + , "- Latest candidate compatible with the set compiler: **" <> if pending.candidateCompilerCompatible then "yes**" else "no**" + , renderRelations "Held back by" pending.blockers + , renderRelations "Candidate blocked by" pending.blockedBy + ] + + renderRelations label relations + | Array.null relations = "- " <> label <> ": none" + | otherwise = "- " <> label <> ":\n" <> String.joinWith "\n" (map (renderRelation >>> (" - " <> _)) relations) + + renderRelation relation = String.joinWith " " + [ "`" <> PackageName.print relation.package <> "@" <> Version.print relation.version <> "` requires" + , "`" <> PackageName.print relation.dependency <> " " <> Range.print relation.range <> "`" + , "but the proposed version is" + , maybe "missing" (\version -> "`" <> Version.print version <> "`") relation.selectedVersion + , "(" <> show relation.packageDirectDependants <> " direct, " <> show relation.packageTransitiveDependants <> " transitive dependants)" + ] diff --git a/scripts/test/Main.purs b/scripts/test/Main.purs index 9bacfbcb..1bf8eba5 100644 --- a/scripts/test/Main.purs +++ b/scripts/test/Main.purs @@ -2,15 +2,21 @@ module Test.Registry.Scripts.Main (main) where import Registry.App.Prelude +import Codec.JSON.DecodeError as CJ.DecodeError +import Data.Array as Array import Data.DateTime (DateTime) import Data.Map as Map +import Data.Set as Set import Data.Time.Duration (Hours(..)) +import Registry.ManifestIndex (IncludeRanges(..), ManifestIndex) +import Registry.ManifestIndex as ManifestIndex import Registry.Metadata (Metadata) import Registry.Metadata as Metadata import Registry.PackageName (PackageName) import Registry.PackageSet (PackageSet) import Registry.PackageSet as PackageSet import Registry.Scripts.PackageSetUpdater as PackageSetUpdater +import Registry.Scripts.VersionCheck as VersionCheck import Registry.Test.Assert as Assert import Registry.Test.Utils as Utils import Registry.Version (Version) @@ -61,6 +67,78 @@ main = runSpecAndExitProcess [ consoleReporter ] do PackageSetUpdater.selectPackageSetCandidates (PackageSetUpdater.RecentUploads now (Hours 24.0)) packageSet metadata `Assert.shouldEqual` expected + Spec.describe "VersionCheck" do + Spec.it "plans coordinated updates, falls back to a compatible intermediate version, and reports blockers" do + let fixtures = versionCheckFixtures + let generatedAt = Utils.unsafeDateTime "2024-01-10T00:00:00.000Z" + let report = VersionCheck.buildReport generatedAt fixtures.packageSet fixtures.metadata fixtures.manifests + let + expectedPlan = Map.fromFoldable + [ Tuple (Utils.unsafePackageName "alpha") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "beta") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "gamma") (Utils.unsafeVersion "1.5.0") + ] + + report.plan.error `Assert.shouldEqual` Nothing + report.plan.packages `Assert.shouldEqual` expectedPlan + report.plan.searchTruncated `Assert.shouldEqual` false + + let gamma = Utils.fromJust "Missing gamma report." $ Array.find (_.name >>> eq (Utils.unsafePackageName "gamma")) report.pending + gamma.candidate `Assert.shouldEqual` Utils.unsafeVersion "2.0.0" + gamma.planned `Assert.shouldEqual` Just (Utils.unsafeVersion "1.5.0") + gamma.staleSince `Assert.shouldEqual` Utils.unsafeDateTime "2023-12-01T00:00:00.000Z" + gamma.staleDays `Assert.shouldEqual` 40 + gamma.directDependants `Assert.shouldEqual` 2 + gamma.transitiveDependants `Assert.shouldEqual` 3 + map _.package gamma.blockers `Assert.shouldEqual` [ Utils.unsafePackageName "consumer", Utils.unsafePackageName "delta" ] + gamma.blockedBy `Assert.shouldEqual` [] + + let consumer = Utils.fromJust "Missing consumer report." $ Array.find (_.name >>> eq (Utils.unsafePackageName "consumer")) report.pending + map _.dependency consumer.blockedBy `Assert.shouldEqual` [ Utils.unsafePackageName "gamma" ] + + case parseJson VersionCheck.reportCodec (printJson VersionCheck.reportCodec report) of + Left error -> Assert.fail $ CJ.DecodeError.print error + Right decoded -> decoded `Assert.shouldEqual` report + + Spec.it "uses the latest version when equally-sized plans are available" do + let fixtures = versionCheckFixtures + let planned = VersionCheck.planPackageSet fixtures.packageSet fixtures.metadata fixtures.manifests + planned `Assert.shouldEqual` Right + { packages: Map.fromFoldable + [ Tuple (Utils.unsafePackageName "alpha") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "beta") (Utils.unsafeVersion "2.0.0") + , Tuple (Utils.unsafePackageName "gamma") (Utils.unsafeVersion "1.5.0") + ] + , searchTruncated: false + } + + Spec.it "preserves existing range drift without allowing an update to introduce it" do + let + packageSet = mkPackageSet $ Map.fromFoldable + [ Tuple (Utils.unsafePackageName "dependency") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "independent") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "legacy") (Utils.unsafeVersion "1.0.0") + ] + metadata = Map.fromFoldable + [ Utils.unsafeMetadata "dependency" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "3.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "independent" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "legacy" [ Tuple "1.0.0" [ "0.15.10" ] ] + ] + manifestSet = Set.fromFoldable + [ Utils.unsafeManifest "dependency" "1.0.0" [] + , Utils.unsafeManifest "dependency" "3.0.0" [] + , Utils.unsafeManifest "independent" "1.0.0" [] + , Utils.unsafeManifest "independent" "2.0.0" [] + , Utils.unsafeManifest "legacy" "1.0.0" [ Tuple "dependency" ">=2.0.0 <3.0.0" ] + ] + manifests = Utils.fromRight "Could not build range-drift manifest index." $ ManifestIndex.fromSet IgnoreRanges manifestSet + + VersionCheck.planPackageSet packageSet metadata manifests + `Assert.shouldEqual` Right + { packages: Map.singleton (Utils.unsafePackageName "independent") (Utils.unsafeVersion "2.0.0") + , searchTruncated: false + } + mkPackageSet :: Map PackageName Version -> PackageSet mkPackageSet packages = PackageSet.PackageSet { version: Utils.unsafeVersion "1.0.0" @@ -73,3 +151,53 @@ withPublishedTime :: forall a. DateTime -> Tuple a Metadata -> Tuple a Metadata withPublishedTime publishedTime (Tuple name (Metadata.Metadata metadata)) = Tuple name $ Metadata.Metadata $ metadata { published = map (_ { publishedTime = publishedTime }) metadata.published } + +type VersionCheckFixtures = + { manifests :: ManifestIndex + , metadata :: Map PackageName Metadata + , packageSet :: PackageSet + } + +versionCheckFixtures :: VersionCheckFixtures +versionCheckFixtures = do + let + packageSet = mkPackageSet $ Map.fromFoldable + [ Tuple (Utils.unsafePackageName "alpha") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "beta") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "consumer") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "delta") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "epsilon") (Utils.unsafeVersion "1.0.0") + , Tuple (Utils.unsafePackageName "gamma") (Utils.unsafeVersion "1.0.0") + ] + metadata = Map.fromFoldable + [ Utils.unsafeMetadata "alpha" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "beta" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "consumer" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "delta" [ Tuple "1.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "epsilon" [ Tuple "1.0.0" [ "0.15.10" ] ] + , Utils.unsafeMetadata "gamma" [ Tuple "1.0.0" [ "0.15.10" ], Tuple "1.5.0" [ "0.15.10" ], Tuple "2.0.0" [ "0.15.10" ] ] + # withVersionPublishedTime "1.5.0" "2023-12-01T00:00:00.000Z" + # withVersionPublishedTime "2.0.0" "2024-01-05T00:00:00.000Z" + ] + manifestSet = Set.fromFoldable + [ Utils.unsafeManifest "alpha" "1.0.0" [ Tuple "beta" ">=1.0.0 <2.0.0" ] + , Utils.unsafeManifest "alpha" "2.0.0" [ Tuple "beta" ">=2.0.0 <3.0.0" ] + , Utils.unsafeManifest "beta" "1.0.0" [] + , Utils.unsafeManifest "beta" "2.0.0" [] + , Utils.unsafeManifest "consumer" "1.0.0" [ Tuple "gamma" ">=1.0.0 <2.0.0" ] + , Utils.unsafeManifest "consumer" "2.0.0" [ Tuple "gamma" ">=2.0.0 <3.0.0" ] + , Utils.unsafeManifest "delta" "1.0.0" [ Tuple "gamma" ">=1.0.0 <2.0.0" ] + , Utils.unsafeManifest "epsilon" "1.0.0" [ Tuple "delta" ">=1.0.0 <2.0.0" ] + , Utils.unsafeManifest "gamma" "1.0.0" [] + , Utils.unsafeManifest "gamma" "1.5.0" [] + , Utils.unsafeManifest "gamma" "2.0.0" [] + ] + manifests = Utils.fromRight "Could not build version-check manifest index." $ ManifestIndex.fromSet IgnoreRanges manifestSet + { manifests, metadata, packageSet } + +withVersionPublishedTime :: forall a. String -> String -> Tuple a Metadata -> Tuple a Metadata +withVersionPublishedTime version publishedTime (Tuple name (Metadata.Metadata metadata)) = do + let parsedVersion = Utils.unsafeVersion version + let parsedTime = Utils.unsafeDateTime publishedTime + Tuple name $ Metadata.Metadata $ metadata + { published = Map.update (Just <<< (_ { publishedTime = parsedTime })) parsedVersion metadata.published } From 4800690d3893b62fb6739b16f5616b668658e69c Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Tue, 28 Jul 2026 00:11:31 +0000 Subject: [PATCH 6/8] Automate compile-guided package set updates --- .github/workflows/version-check.yml | 226 ++++++++- SPEC.md | 37 +- app-e2e/src/Test/E2E/Scripts.purs | 30 +- app/src/App/API.purs | 32 +- app/src/App/GitHubIssue.purs | 15 +- app/src/App/Server/Router.purs | 3 +- app/test/App/API.purs | 147 +++++- lib/src/Operation.purs | 54 ++- lib/test/Registry/Operation.purs | 18 + scripts/spago.yaml | 1 + scripts/src/PackageSetPlanner.purs | 706 ++++++++++++++++++++++++++++ scripts/src/PackageSetUpdater.purs | 217 +++++++-- scripts/src/VersionCheck.purs | 533 ++++++++++----------- scripts/test/Main.purs | 411 +++++++++++----- spago.lock | 1 + 15 files changed, 1886 insertions(+), 545 deletions(-) create mode 100644 scripts/src/PackageSetPlanner.purs diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml index e1a9b5f1..6d54692b 100644 --- a/.github/workflows/version-check.yml +++ b/.github/workflows/version-check.yml @@ -1,8 +1,8 @@ -name: Weekly package set version check +name: Daily package set planning on: schedule: - - cron: '0 6 * * 1' + - cron: '0 6 * * *' workflow_dispatch: permissions: @@ -10,13 +10,16 @@ permissions: issues: write concurrency: - group: weekly-package-set-version-check + group: package-set-version-check cancel-in-progress: false jobs: version-check: runs-on: ubuntu-latest - timeout-minutes: 90 + # Planning performs at most 96 incremental compile probes, then a submitted + # server job is polled for at most 60 minutes. Leave enough headroom that a + # difficult plan can still emit its report and reconcile trustee issues. + timeout-minutes: 180 steps: - name: Checkout code @@ -30,33 +33,206 @@ jobs: with: use-flakehub: false - - name: Plan and verify pending package set updates - run: nix run .#version-check -- --verify --output version-check.json --markdown version-check.md + - name: Plan, verify, and submit pending package set updates + run: nix run .#version-check -- --verify --submit --output version-check.json --markdown version-check.md - name: Upload report + if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: package-set-version-check path: | version-check.json version-check.md + if-no-files-found: warn + retention-days: 14 - - name: Update rolling report issue - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - run: | - set -euo pipefail - - title='Weekly package set version report' - issue_number=$( - gh issue list --repo "$REPOSITORY" --state open --search "$title in:title" --limit 100 --json number,title \ - | jq -r --arg title "$title" '.[] | select(.title == $title) | .number' \ - | sed -n '1p' - ) - - if [[ -n "$issue_number" ]]; then - gh issue edit "$issue_number" --repo "$REPOSITORY" --body-file version-check.md - else - gh issue create --repo "$REPOSITORY" --title "$title" --body-file version-check.md - fi + - name: Reconcile rolling report and trustee decisions + if: ${{ !cancelled() }} + uses: actions/github-script@v8 + with: + github-token: ${{ github.token }} + script: | + const fs = require("node:fs"); + + const expectedRepository = "purescript/registry-dev"; + const repository = `${context.repo.owner}/${context.repo.repo}`; + if (repository !== expectedRepository) { + throw new Error(`Refusing to manage issues in ${repository}; expected ${expectedRepository}.`); + } + + const summaryMarker = ""; + const decisionMarker = ""; + const componentMarkerPrefix = ""; + const summaryTitle = "Package set planning report"; + + const fail = message => { throw new Error(message); }; + const isObject = value => value !== null && typeof value === "object" && !Array.isArray(value); + const normalize = value => (value || "").replace(/\r\n/g, "\n"); + const lines = value => normalize(value).split("\n"); + const hasExactMarker = (issue, marker) => lines(issue.body).includes(marker); + const markerFor = (prefix, value) => `${prefix}${value}${markerSuffix}`; + const hasReservedMarker = value => lines(value).some(line => line.startsWith(""; - const decisionMarker = ""; - const componentMarkerPrefix = ""; - const summaryTitle = "Package set planning report"; - - const fail = message => { throw new Error(message); }; - const isObject = value => value !== null && typeof value === "object" && !Array.isArray(value); - const normalize = value => (value || "").replace(/\r\n/g, "\n"); - const lines = value => normalize(value).split("\n"); - const hasExactMarker = (issue, marker) => lines(issue.body).includes(marker); - const markerFor = (prefix, value) => `${prefix}${value}${markerSuffix}`; - const hasReservedMarker = value => lines(value).some(line => line.startsWith("" + +decisionMarker :: String +decisionMarker = "" + +componentMarkerPrefix :: String +componentMarkerPrefix = "" + +summaryTitle :: String +summaryTitle = "Package set planning report" + +reconcile :: Octokit -> ReconciliationInput -> Aff (Either String Unit) +reconcile octokit input = + Octokit.request octokit (Octokit.listIssuesRequest registryDev) >>= case _ of + Left error -> pure $ Left $ "Could not list registry-dev issues: " <> Octokit.printGitHubError error + Right allItems -> case planIssueReconciliation (input { reconcileDecisions = false }) allItems of + Left error -> pure $ Left error + Right summaryMutations -> applyMutations summaryMutations >>= case _ of + Left error -> pure $ Left error + Right _ + | not input.reconcileDecisions -> pure $ Right unit + | otherwise -> case planIssueReconciliation input allItems of + Left error -> pure $ Left error + Right mutations -> applyMutations $ Array.filter (not <<< isSummaryMutation) mutations + where + isSummaryMutation = case _ of + CreateIssue issue -> isJust $ String.stripPrefix (String.Pattern summaryMarker) issue.body + UpdateIssue issue -> isJust $ String.stripPrefix (String.Pattern summaryMarker) issue.body + + applyMutations mutations = case Array.uncons mutations of + Nothing -> pure $ Right unit + Just { head: mutation, tail } -> do + result <- case mutation of + CreateIssue issue -> Octokit.request octokit $ Octokit.createIssueRequest + { address: registryDev, body: issue.body, title: issue.title } + UpdateIssue issue -> Octokit.request octokit $ Octokit.updateIssueRequest + { address: registryDev, body: issue.body, issue: issue.issue, state: issue.state, title: issue.title } + case result of + Left error -> pure $ Left $ "Could not reconcile registry-dev issues: " <> Octokit.printGitHubError error + Right _ -> applyMutations tail + +planIssueReconciliation :: ReconciliationInput -> Array Issue -> Either String (Array IssueMutation) +planIssueReconciliation input allItems = do + let markdown = trimTrailingNewlines $ normalize input.markdown + when (hasReservedMarker markdown) $ Left "The generated Markdown contains a reserved automation marker." + let summaryBody = summaryMarker <> "\n\n" <> markdown <> "\n" + when (CodeUnits.length summaryBody > 65_536) $ Left "The rolling report issue body is larger than 65536 characters." + + let allIssues = Array.filter (not <<< _.isPullRequest) allItems + let summaryIssues = Array.filter (flip hasExactMarker summaryMarker) allIssues + when (Array.length summaryIssues > 1) $ Left "Found multiple rolling issues with the exact summary marker." + when (any (hasDuplicateMarker summaryMarker) summaryIssues) $ Left "The rolling issue has duplicate summary markers." + when (any (flip hasExactMarker decisionMarker) summaryIssues) $ Left "An issue contains both summary and decision markers." + + let + summaryMutations = case Array.head summaryIssues of + Nothing -> [ CreateIssue { title: summaryTitle, body: summaryBody } ] + Just issue + | issue.state /= "open" || issue.title /= summaryTitle || normalize (fromMaybe "" issue.body) /= summaryBody -> + [ UpdateIssue { issue: issue.number, title: summaryTitle, body: summaryBody, state: "open" } ] + Just _ -> [] + + if not input.reconcileDecisions then pure summaryMutations + else do + planned <- traverse prepareIntervention input.interventions + let components = map _.component planned + let fingerprints = map _.fingerprint planned + when (Set.size (Set.fromFoldable components) /= Array.length components) $ Left "Manual interventions contain duplicate component identifiers." + when (Set.size (Set.fromFoldable fingerprints) /= Array.length fingerprints) $ Left "Manual interventions contain duplicate fingerprints." + + let decisionIssues = Array.filter (flip hasExactMarker decisionMarker) allIssues + when (any (flip hasExactMarker summaryMarker) decisionIssues) $ Left "An issue contains both summary and decision markers." + indexed <- indexDecisionIssues decisionIssues + + let + activeMutations = planned >>= \intervention -> case Map.lookup intervention.component indexed.byComponent of + Nothing -> [ CreateIssue { title: intervention.title, body: intervention.body } ] + Just issue + | issue.state /= "open" || issue.title /= intervention.title || normalize (fromMaybe "" issue.body) /= intervention.body -> + [ UpdateIssue { issue: issue.number, title: intervention.title, body: intervention.body, state: "open" } ] + Just _ -> [] + + for_ planned \intervention -> case Map.lookup intervention.fingerprint indexed.byFingerprint of + Just owner | Just owner /= Map.lookup intervention.component indexed.byComponent -> + Left $ "Fingerprint " <> intervention.fingerprint <> " belongs to a decision issue for another component." + _ -> pure unit + + pure $ summaryMutations <> activeMutations + +type PreparedIntervention = + { body :: String + , component :: String + , fingerprint :: String + , title :: String + } + +prepareIntervention :: Intervention -> Either String PreparedIntervention +prepareIntervention intervention = do + unless (safeMarkerValue 128 intervention.component) $ Left "A manual intervention has an unsafe component identifier." + unless (safeMarkerValue 200 intervention.fingerprint) $ Left "A manual intervention has an unsafe fingerprint." + when + ( String.trim intervention.title /= intervention.title + || String.contains (String.Pattern "\n") intervention.title + || String.contains (String.Pattern "\r") intervention.title + || CodeUnits.length intervention.title + == 0 + || CodeUnits.length intervention.title + > 256 + ) + $ Left "A manual intervention has an invalid issue title." + when (String.trim intervention.markdownBody == "" || hasReservedMarker intervention.markdownBody) $ + Left "A manual intervention has invalid Markdown." + + let payloadHeading = if intervention.removalVerified then "## Compile-verified removal payload" else "## Diagnostic payload (not compile-verified)" + let + payloadNote = + if intervention.removalVerified then + "The planner atomically verified this package-set payload. A trustee must review it before taking action." + else + "The planner did not find a compile-verified removal for this component. This payload is diagnostic only and must not be applied as a verified plan." + let + body = String.joinWith "\n" + [ decisionMarker + , markerFor componentMarkerPrefix intervention.component + , markerFor fingerprintMarkerPrefix intervention.fingerprint + , "" + , trimTrailingNewlines $ normalize intervention.markdownBody + , "" + , payloadHeading + , "" + , payloadNote + , "" + , "```json" + , intervention.payload + , "```" + ] + when (CodeUnits.length body > 65_536) $ Left "A manual intervention issue body is larger than 65536 characters." + pure { body, component: intervention.component, fingerprint: intervention.fingerprint, title: intervention.title } + +type IndexedIssues = + { byComponent :: Map String Issue + , byFingerprint :: Map String Issue + } + +indexDecisionIssues :: Array Issue -> Either String IndexedIssues +indexDecisionIssues = Foldable.foldM insert { byComponent: Map.empty, byFingerprint: Map.empty } + where + insert indexed issue = do + when (hasDuplicateMarker decisionMarker issue) $ Left "A decision issue has duplicate decision markers." + component <- oneMarker componentMarkerPrefix issue + fingerprint <- oneMarker fingerprintMarkerPrefix issue + unless (safeMarkerValue 128 component && safeMarkerValue 200 fingerprint) $ Left "A decision issue has invalid identity markers." + when (Map.member component indexed.byComponent) $ Left "Multiple decision issues have the same component marker." + when (Map.member fingerprint indexed.byFingerprint) $ Left "Multiple decision issues have the same fingerprint marker." + pure + { byComponent: Map.insert component issue indexed.byComponent + , byFingerprint: Map.insert fingerprint issue indexed.byFingerprint + } + +oneMarker :: String -> Issue -> Either String String +oneMarker prefix issue = case markers prefix issue of + [ value ] | markerFor prefix value `Array.elem` issueLines issue -> pure value + _ -> Left "A decision issue has malformed identity markers." + +markers :: String -> Issue -> Array String +markers prefix issue = Array.mapMaybe extract (issueLines issue) + where + extract line = String.stripPrefix (String.Pattern prefix) line >>= String.stripSuffix (String.Pattern markerSuffix) + +markerFor :: String -> String -> String +markerFor prefix value = prefix <> value <> markerSuffix + +safeMarkerValue :: Int -> String -> Boolean +safeMarkerValue limit value = + CodeUnits.length value > 0 + && CodeUnits.length value + <= limit + && not (String.contains (String.Pattern "\n") value) + && not (String.contains (String.Pattern "\r") value) + && not (String.contains (String.Pattern "") value) + +hasReservedMarker :: String -> Boolean +hasReservedMarker = String.split (String.Pattern "\n") + >>> any (isJust <<< String.stripPrefix (String.Pattern "