diff --git a/.github/workflows/check-testnet-manifest.yml b/.github/workflows/check-testnet-manifest.yml index fb6825c66d5..eb504d686e3 100644 --- a/.github/workflows/check-testnet-manifest.yml +++ b/.github/workflows/check-testnet-manifest.yml @@ -34,3 +34,6 @@ jobs: - name: Validate cardano-testnet manifest golden file against its schema run: make testnet-manifest-validate + + - name: Validate cardano-testnet status golden files against their schema + run: make testnet-status-validate diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index b0457ab4d4f..f4fef6cd1c5 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -142,7 +142,7 @@ jobs: run: | cabal test cardano-testnet cardano-node cardano-node-chairman cardano-submit-api - - name: Validate produced manifest files against schema + - name: Validate produced manifest and status output against schemas if: matrix.validate-manifest run: | # This job has no nix, that is why we pin the version with pipx separately. @@ -159,6 +159,10 @@ jobs: if ! kill -0 "$testnet_pid" 2>/dev/null; then break; fi sleep 5 done + # Exit code 0: the chain is producing blocks. + status_rc=0 + "$(cabal list-bin cardano-testnet)" status --testnet-dir "$dir" --json > "$RUNNER_TEMP/status.json" || status_rc=$? + cat "$RUNNER_TEMP/status.json" # SIGINT, not SIGTERM: the harness tears its nodes down on SIGINT only # (see Testnet.Signal.interruptNodesOnSigINT). kill -INT "$testnet_pid" 2>/dev/null || true @@ -169,6 +173,11 @@ jobs: exit 1 fi check-jsonschema --schemafile cardano-testnet/schemas/manifest.schema.json "$dir/manifest.json" + if [ "$status_rc" != 0 ]; then + echo "ERROR: cardano-testnet status exited with $status_rc" + exit 1 + fi + check-jsonschema --schemafile cardano-testnet/schemas/status.schema.json "$RUNNER_TEMP/status.json" - name: Tar failed tests workspaces if: ${{ failure() }} diff --git a/Makefile b/Makefile index 03753373428..7a4fef2be4f 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,12 @@ testnet-manifest-validate: ## Validate manifest golden file (and optionally MANI cardano-testnet/test/cardano-testnet-golden/files/golden/manifest.json $(if $(MANIFEST),nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/manifest.schema.json $(MANIFEST),) +testnet-status-validate: ## Validate status golden files (and optionally STATUS=path) against their schema + nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/status.schema.json \ + cardano-testnet/test/cardano-testnet-golden/files/golden/status.json \ + cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json + $(if $(STATUS),nix run .#check-jsonschema -- --schemafile cardano-testnet/schemas/status.schema.json $(STATUS),) + ### ### Workbench: cluster shells ### @@ -84,6 +90,6 @@ cls: .PHONY: help lint hlint host-hlint haddock-hoogle stylish-haskell cabal-hashes cli node \ trace-documentation trace-schemas-regenerate trace-schemas-overrides-check \ trace-schemas-overrides-coverage trace-schemas-validate \ - testnet-manifest-validate \ + testnet-manifest-validate testnet-status-validate \ workbench-ci ci ci-report ci-targets \ clean full-clean cls diff --git a/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml new file mode 100644 index 00000000000..a9997c87a24 --- /dev/null +++ b/cardano-testnet/.changes/20261020_193638_cardano-testnet_pablo.lamela_status_command.yml @@ -0,0 +1,10 @@ +description: Added `status` command that checks the status of a testnet created with + `cardano-testnet`. The output can be either in human readable format or JSON (if + the `--json` flag is specified). The library also exports the `Testnet.Status.Types`, + `Testnet.Status.Run` and `Testnet.Status.Parsers` modules, `isProcessAlive` from + `Testnet.Signal`, `testnetEpochSlots` from `Testnet.Types` and `defaultTestnetDir` + from `Testnet.Start.Types`. +kind: +- feature +pr: 6741 +project: cardano-testnet diff --git a/cardano-testnet/README.md b/cardano-testnet/README.md index f9fb0cd0e60..963df286e21 100644 --- a/cardano-testnet/README.md +++ b/cardano-testnet/README.md @@ -53,7 +53,7 @@ export CARDANO_NODE=path to your executable --> ```text -Usage: cardano-testnet (cardano | create-env | version | help) +Usage: cardano-testnet (cardano | create-env | status | version | help) Available options: -h,--help Show this help text @@ -61,6 +61,7 @@ Available options: Available commands: cardano Start a testnet and keep it running until stopped create-env Create a sandbox for Cardano testnet + status Check the status of a running testnet version Show cardano-testnet version help Show cardano-testnet help ``` @@ -442,6 +443,36 @@ readers never see a partial file. It is written once and never modified. A pre-existing manifest from an earlier run in the same directory is deleted before nodes start. +### Status output + +`cardano-testnet status --json` prints the health of a testnet as a single +JSON object: the state of the network, the exit code of the command, why +there is no testnet (when there is none), the +freshest tip, and the state and tip of every node, with the error of a node +that did not answer. The exit code is 0 while +the chain is producing blocks, even when some nodes are not healthy. A script +that needs every node to be healthy can check the `status` field: + +```bash +[ "$(cardano-testnet status --testnet-dir testnet --json | jq -r '.status')" = running ] +``` + +The exit code, for the table and for `--json`: + +| Exit code | Meaning | +|-----------|-----------------------------------------------------------| +| 0 | `running` or `degraded`: the chain is producing blocks | +| 1 | the command failed, for example on an unreadable manifest | +| 3 | `stalled`: some nodes answer, but none has a fresh tip | +| 4 | `stopped`: no node answers | +| 5 | `no-testnet`: there is no manifest in the directory | + +**Stability promise.** Like the manifest, the output has a strict JSON Schema +(`cardano-testnet/schemas/status.schema.json`, draft 2020-12, unknown fields +are an error) that lives in the repository and is the contract. Any change to +the output shape, including new fields, bumps the schema version, so the +output always matches the schema of its own version exactly. + ## Supported versions cardano-testnet is designed to run the cardano-cli and cardano-node that ship diff --git a/cardano-testnet/app/cardano-testnet.hs b/cardano-testnet/app/cardano-testnet.hs index 53de472de9a..f1f9776dbb9 100644 --- a/cardano-testnet/app/cardano-testnet.hs +++ b/cardano-testnet/app/cardano-testnet.hs @@ -4,11 +4,14 @@ import Cardano.CLI.Environment (getEnvCli) import qualified Cardano.Crypto.Init as Crypto import qualified Options.Applicative as Opt +import System.IO (BufferMode (LineBuffering), hSetBuffering, stderr, stdout) import Parsers.Run (opts, pref, runTestnetCmd) main :: IO () main = do + hSetBuffering stdout LineBuffering + hSetBuffering stderr LineBuffering Crypto.cryptoInit envCli <- getEnvCli diff --git a/cardano-testnet/cardano-testnet.cabal b/cardano-testnet/cardano-testnet.cabal index 94f346fddea..ec160e57261 100644 --- a/cardano-testnet/cardano-testnet.cabal +++ b/cardano-testnet/cardano-testnet.cabal @@ -132,6 +132,9 @@ library Testnet.Start.Byron Testnet.Start.Cardano Testnet.Start.Types + Testnet.Status.Parsers + Testnet.Status.Run + Testnet.Status.Types Testnet.SubmitApi Testnet.TestQueryCmds Testnet.Types @@ -171,6 +174,7 @@ test-suite cardano-testnet-golden other-modules: Cardano.Testnet.Test.Golden.Config Cardano.Testnet.Test.Golden.Help Cardano.Testnet.Test.Golden.Manifest + Cardano.Testnet.Test.Golden.Status Cardano.Testnet.Test.Golden.Util type: exitcode-stdio-1.0 @@ -255,6 +259,8 @@ test-suite cardano-testnet-test Cardano.Testnet.Test.Node.Shutdown Cardano.Testnet.Test.Parser Cardano.Testnet.Test.Manifest + Cardano.Testnet.Test.Status + Cardano.Testnet.Test.StatusRules Cardano.Testnet.Test.MainnetParams Cardano.Testnet.Test.SanityCheck Cardano.Testnet.Test.RunTestnet diff --git a/cardano-testnet/schemas/status.schema.json b/cardano-testnet/schemas/status.schema.json new file mode 100644 index 00000000000..a23266064d0 --- /dev/null +++ b/cardano-testnet/schemas/status.schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/IntersectMBO/cardano-node/cardano-testnet/schemas/status.schema.json", + "title": "cardano-testnet status", + "description": "Output of 'cardano-testnet status --json': the health of a testnet, found by asking every node in its manifest for its chain tip. The schema is strict: it describes schema version 1 exactly, unknown fields are an error, and any change to the output shape bumps schemaVersion.", + "type": "object", + "required": [ + "schemaVersion", + "checkedAt", + "status", + "exitCode", + "chainProducingBlocks", + "bestTip", + "nodes", + "noTestnetReason" + ], + "additionalProperties": false, + "properties": { + "schemaVersion": { + "type": "integer", + "const": 1, + "description": "Schema version. Any change to the output shape, including new fields or values, bumps it." + }, + "checkedAt": { + "type": "string", + "format": "date-time", + "description": "UTC time the check started (ISO-8601)." + }, + "status": { + "type": "string", + "enum": ["running", "degraded", "stalled", "stopped", "no-testnet"], + "description": "State of the whole network. \"running\": every node is ok. \"degraded\": some nodes are ok, but not all. \"stalled\": some nodes answer, but none is ok. \"stopped\": no node answers. \"no-testnet\": there is no manifest in the directory." + }, + "noTestnetReason": { + "description": "Why there is no testnet, when the status is \"no-testnet\". \"dir-missing\": the directory does not exist. \"not-ready\": it has genesis files but no manifest: the testnet is still starting, or it never became ready. \"not-a-testnet-dir\": it is not the output directory of a testnet. Null for any other status.", + "oneOf": [ + { "type": "null" }, + { "type": "string", "enum": ["dir-missing", "not-ready", "not-a-testnet-dir"] } + ] + }, + "exitCode": { + "type": "integer", + "enum": [0, 3, 4, 5], + "description": "Exit code of the command: 0 for \"running\" and \"degraded\", 3 for \"stalled\", 4 for \"stopped\", 5 for \"no-testnet\"." + }, + "chainProducingBlocks": { + "type": "boolean", + "description": "True when the exit code is 0: the status is \"running\" or \"degraded\"." + }, + "bestTip": { + "description": "The newest tip of all nodes: the one with the highest slot. Null when no node answered with a tip.", + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/tip" } + ] + }, + "nodes": { + "type": "array", + "description": "One entry per node, in the order of the manifest. Empty for \"no-testnet\".", + "items": { + "type": "object", + "required": ["name", "role", "pid", "pidAlive", "state", "tip", "error"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Node name, as in the manifest." + }, + "role": { + "type": "string", + "enum": ["spo", "relay"], + "description": "\"spo\" for stake pool operators, \"relay\" for relay nodes, as in the manifest." + }, + "pid": { + "description": "OS process ID from the manifest, or null when the manifest has none.", + "oneOf": [ + { "type": "null" }, + { "type": "integer" } + ] + }, + "pidAlive": { + "description": "Whether a process with this pid exists, or null when that is unknown: no pid, or Windows.", + "oneOf": [ + { "type": "null" }, + { "type": "boolean" } + ] + }, + "state": { + "type": "string", + "enum": ["ok", "stalled", "unreachable", "down"], + "description": "\"ok\": it answers, and its tip is fresh. \"stalled\": it answers, but its tip is too old, or it has no blocks yet. \"unreachable\": it does not answer, but its process may be alive. \"down\": it does not answer, and its process is gone." + }, + "tip": { + "description": "The node's tip, or null when it did not answer with one.", + "oneOf": [ + { "type": "null" }, + { "$ref": "#/$defs/tip" } + ] + }, + "error": { + "description": "Why the node did not answer (an error, or no answer in time), or null when it answered.", + "oneOf": [ + { "type": "null" }, + { "type": "string" } + ] + } + } + } + } + }, + "$defs": { + "tip": { + "type": "object", + "required": ["slot", "blockNo", "hash", "ageSeconds"], + "additionalProperties": false, + "properties": { + "slot": { + "type": "integer", + "minimum": 0, + "description": "Slot number of the tip." + }, + "blockNo": { + "type": "integer", + "minimum": 0, + "description": "Block number of the tip." + }, + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$", + "description": "Block header hash of the tip, hex-encoded." + }, + "ageSeconds": { + "type": "number", + "description": "Seconds from the start of the tip's slot until the node answered." + } + } + } + } +} diff --git a/cardano-testnet/src/Parsers/Run.hs b/cardano-testnet/src/Parsers/Run.hs index d3d2a9ed27d..0d36c211be9 100644 --- a/cardano-testnet/src/Parsers/Run.hs +++ b/cardano-testnet/src/Parsers/Run.hs @@ -27,6 +27,9 @@ import System.Directory (doesDirectoryExist) import Testnet.Filepath (unTmpAbsPath) import Testnet.Start.Cardano import Testnet.Start.Types +import Testnet.Status.Parsers (cmdCheckStatus) +import Testnet.Status.Run (runCheckStatusOptions) +import Testnet.Status.Types (CheckStatusOptions) import Testnet.Types (TestnetNode (..)) import Parsers.Cardano @@ -62,6 +65,7 @@ opts envCli = Opt.info (commands envCli <**> helper) idm data CardanoTestnetCommands = StartCardanoTestnet CardanoTestnetCliOptions | CreateTestnetEnv CardanoTestnetCreateEnvOptions + | CheckStatus CheckStatusOptions | GetVersion VersionOptions | Help ParserPrefs (ParserInfo CardanoTestnetCommands) HelpOptions @@ -70,6 +74,7 @@ commands envCli = asum [ fmap StartCardanoTestnet (subparser cmdCardano) , fmap CreateTestnetEnv (subparser cmdCreateEnv) + , fmap CheckStatus (subparser cmdCheckStatus) , fmap GetVersion (subparser cmdVersion) , fmap (Help pref (opts envCli)) (subparser cmdHelp) ] @@ -79,6 +84,7 @@ runTestnetCmd :: CardanoTestnetCommands -> IO () runTestnetCmd = \case StartCardanoTestnet cmdOpts -> runCardanoOptions cmdOpts CreateTestnetEnv cmdOpts -> createEnvOptions cmdOpts + CheckStatus cmdOpts -> runCheckStatusOptions cmdOpts GetVersion cmdOpts -> runVersionOptions cmdOpts Help pPrefs pInfo cmdOpts -> runHelpOptions pPrefs pInfo cmdOpts @@ -96,7 +102,7 @@ runCardanoOptions = \case -- Create the sandbox, then run cardano-testnet. -- It is not necessary to update timestamps here, because -- the genesis files will be created with up-to-date stamps already. - let dirName = fromMaybe "testnet" noEnvOutputDir + let dirName = fromMaybe defaultTestnetDir noEnvOutputDir conf <- mkConfigAbs dirName runSimpleApp . runResourceT $ do logInfo $ "Creating environment: " <> display (tempAbsPath conf) diff --git a/cardano-testnet/src/Testnet/Signal.hs b/cardano-testnet/src/Testnet/Signal.hs index 5523da03993..b992a85921e 100644 --- a/cardano-testnet/src/Testnet/Signal.hs +++ b/cardano-testnet/src/Testnet/Signal.hs @@ -10,15 +10,17 @@ module Testnet.Signal ( hardKillProcess , interruptNodesOnSigINT + , isProcessAlive ) where #ifdef UNIX import Control.Monad -import System.Posix.Signals (Handler (..), installHandler, raiseSignal, sigINT, - sigKILL, signalProcess) -import System.Process (ProcessHandle, getPid, interruptProcessGroupOf) +import System.IO.Error (isDoesNotExistError, tryIOError) +import System.Posix.Signals (Handler (..), installHandler, nullSignal, raiseSignal, + sigINT, sigKILL, signalProcess) +import System.Process (Pid, ProcessHandle, getPid, interruptProcessGroupOf) #else -import System.Process (ProcessHandle, terminateProcess) +import System.Process (Pid, ProcessHandle, terminateProcess) #endif import Data.List.NonEmpty (NonEmpty) @@ -49,3 +51,20 @@ hardKillProcess hProcess = getPid hProcess >>= mapM_ (signalProcess sigKILL) #else hardKillProcess = terminateProcess #endif + +-- | Whether a process with the given pid exists: 'Just' the answer on unix, +-- where signal 0 checks this without sending anything, and 'Nothing' on +-- Windows. A dead process that nobody has waited for yet (a zombie) still +-- exists, and so does an unrelated process that reuses the pid. +isProcessAlive :: Pid -> IO (Maybe Bool) +#ifdef UNIX +isProcessAlive pid + | pid <= 0 = pure (Just False) -- 0 and negative pids name process groups, not one process + | otherwise = do + result <- tryIOError $ signalProcess nullSignal pid + pure . Just $ case result of + Left e | isDoesNotExistError e -> False -- no such process + _ -> True -- signalled, or not allowed to: either way it exists +#else +isProcessAlive _ = pure Nothing +#endif diff --git a/cardano-testnet/src/Testnet/Start/Types.hs b/cardano-testnet/src/Testnet/Start/Types.hs index adfa075aa33..66667e4faed 100644 --- a/cardano-testnet/src/Testnet/Start/Types.hs +++ b/cardano-testnet/src/Testnet/Start/Types.hs @@ -47,6 +47,7 @@ module Testnet.Start.Types , NodeConfigurationYaml , mkConf , mkConfigAbs + , defaultTestnetDir ) where import Cardano.Api hiding (cardanoEra) @@ -361,6 +362,11 @@ mkConfig tempAbsPath' = , updateTimestamps = DontUpdateTimestamps } +-- | The directory of the testnet when none is given: the @cardano@ command +-- creates it, and the @status@ command looks into it. +defaultTestnetDir :: FilePath +defaultTestnetDir = "./testnet" + -- | Create a 'Conf' from an absolute path, with updating time stamps disabled. mkConfigAbs :: FilePath -> IO Conf mkConfigAbs userOutputDir = do diff --git a/cardano-testnet/src/Testnet/Status/Parsers.hs b/cardano-testnet/src/Testnet/Status/Parsers.hs new file mode 100644 index 00000000000..589195fc5b3 --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Parsers.hs @@ -0,0 +1,52 @@ +module Testnet.Status.Parsers + ( cmdCheckStatus + ) where + +import Cardano.CLI.EraBased.Common.Option (command') +import Cardano.Prelude (Nat) + +import Prelude + +import Control.Monad (when) +import Data.Default.Class (def) +import Options.Applicative (CommandFields, Mod, Parser) +import qualified Options.Applicative as OA + +import Testnet.Start.Types (defaultTestnetDir) +import Testnet.Status.Types (CheckStatusOptions (..), OutputFormat (..)) + +cmdCheckStatus :: Mod CommandFields CheckStatusOptions +cmdCheckStatus = command' "status" "Check the status of a running testnet" optsCheckStatus + +optsCheckStatus :: Parser CheckStatusOptions +optsCheckStatus = CheckStatusOptions + <$> pOutputDir + <*> pStatusCheckTimeout + <*> pOutputFormat + +pOutputDir :: Parser (Maybe FilePath) +pOutputDir = OA.optional (OA.strOption + ( OA.long "testnet-dir" + <> OA.metavar "FILEPATH" + <> OA.help ("Path to the environment folder of the testnet whose status to check, by default " + <> show defaultTestnetDir <> ".") + )) + + +pStatusCheckTimeout :: Parser Nat +pStatusCheckTimeout = OA.option timeoutReader + ( OA.long "timeout" + <> OA.metavar "SECONDS" + <> OA.help "How many seconds to wait for each node to answer (1 - 3600)." + <> OA.value 5 + <> OA.showDefault + ) + where + timeoutReader = OA.auto >>= \n -> do + when (n < 1 || n > 3600) $ + OA.readerError $ "Timeout out of range (1 - 3600): " <> show n + pure n + +pOutputFormat :: Parser OutputFormat +pOutputFormat = + OA.flag def OutputFormatJson (OA.long "json" <> OA.help "Output status as JSON instead of a table.") diff --git a/cardano-testnet/src/Testnet/Status/Run.hs b/cardano-testnet/src/Testnet/Status/Run.hs new file mode 100644 index 00000000000..960434f812c --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Run.hs @@ -0,0 +1,327 @@ +{-# LANGUAGE NumericUnderscores #-} + +module Testnet.Status.Run ( + runCheckStatusOptions, + checkStatus, + calcNodeState, + classifyNetwork, + newestTip, + renderStatusReportHumanReadable, +) where + +import Cardano.Api (BlockNo (..), CardanoEra (..), ChainTip (..), + ConsensusModeParams (..), LocalNodeConnectInfo (..), NetworkId (..), + NetworkMagic (..), ShelleyGenesis, SlotNo (..), getLocalChainTip, mapFile) + +import qualified Cardano.Ledger.Shelley.API as SL +import Cardano.Node.Testnet.Paths (defaultGenesisFilepath, defaultManifestFile) +import Cardano.Prelude (Nat, NonEmpty, Word32, toList, transpose) + +import Prelude + +import Data.Aeson (FromJSON, eitherDecodeFileStrict') +import Data.Aeson.Encode.Pretty (encodePretty) +import qualified Data.ByteString.Lazy as LBS +import Data.Foldable (maximumBy) +import Data.List.NonEmpty (nonEmpty) +import Data.Maybe (fromMaybe, mapMaybe) +import Data.Ord (comparing) +import Data.Time (NominalDiffTime, UTCTime, addUTCTime, diffUTCTime, getCurrentTime) +import GHC.Stack (HasCallStack) +import System.Directory (doesDirectoryExist, doesFileExist) +import System.Exit (exitWith) +import System.FilePath (()) +import System.Timeout (timeout) +import Text.Printf (printf) + +import Testnet.ChainWatchdog (chainForecastHorizon, chainStallTimeoutFromHorizon) +import Testnet.Manifest (Manifest (..), ManifestGenesisFiles (..), ManifestNetwork (..), + ManifestNode (..), ManifestNodeRole (..), ManifestPaths (..)) +import Testnet.Signal (isProcessAlive) +import Testnet.Start.Types (defaultTestnetDir) +import Testnet.Status.Types (CheckStatusOptions (..), NetworkStatus (..), + NoTestnetReason (..), NodeAnswer (..), NodeProbeResult (..), NodeState (..), + OutputFormat (..), StatusReport (..), TipInfo (..), exitCodeForStatus) +import Testnet.Types (sgSlotLength, sgSystemStart, testnetEpochSlots) + +import RIO (displayException, throwString, tryAny) +import UnliftIO.Async (forConcurrently) + +-- | Run the @status@ command: print the report (a table, or JSON) on stdout +-- and exit with 0 (running or degraded), 3 (stalled), 4 (stopped) or 5 (no +-- testnet). When the manifest or the genesis file cannot be read, print the +-- error on stderr and exit with 1. +runCheckStatusOptions :: HasCallStack => CheckStatusOptions -> IO () +runCheckStatusOptions + CheckStatusOptions + { testnetDir = dir + , statusCheckTimeout = timeoutSeconds + , outputFormat = format + } = do + status <- checkStatus (fromMaybe defaultTestnetDir dir) timeoutSeconds + case format of + OutputFormatText -> putStr $ renderStatusReportHumanReadable status + OutputFormatJson -> do LBS.putStr $ encodePretty status + putStrLn "" + exitWith . exitCodeForStatus $ reportStatus status + +-- | Check the testnet in the given output directory. Each node gets the +-- given number of seconds to answer, and all nodes are asked at the same +-- time. It throws when the check itself fails: when the manifest, or the +-- shelley genesis it points to, cannot be read. +checkStatus :: HasCallStack => FilePath -> Nat -> IO StatusReport +checkStatus outputDir timeoutSeconds = do + let manifestFile = outputDir defaultManifestFile + hasManifest <- doesFileExist manifestFile + if not hasManifest + then noTestnetReport outputDir + else do + manifest@Manifest{manifestPaths = ManifestPaths{mpGenesisFiles = ManifestGenesisFiles{mgfShelley = shelleyGenesisFile}}} + <- readJsonFile manifestFile + genesis <- readJsonFile $ outputDir shelleyGenesisFile + checkNodes outputDir timeoutSeconds manifest genesis + where + readJsonFile :: (HasCallStack, FromJSON a) => String -> IO a + readJsonFile file = + either (\e -> throwString $ "cannot decode " <> file <> ": " <> e) pure + =<< eitherDecodeFileStrict' file + +checkNodes :: FilePath -> Nat -> Manifest -> ShelleyGenesis -> IO StatusReport +checkNodes outputDir timeoutSeconds manifest@(Manifest{manifestNetwork = ManifestNetwork{mnMagic = magic}}) genesis = do + checkedAt <- getCurrentTime + nodes <- forConcurrently (manifestNodes manifest) $ + probeNode outputDir timeoutSeconds magic limit slotStart + pure StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = checkedAt + , reportStatus = classifyNetwork (probeNodeState <$> nodes) + , reportBestTip = newestTip . mapMaybe probeNodeTipInfo $ toList nodes + , reportNodes = toList nodes + , reportProblemExplanation = Nothing + } + where + -- The chain watchdog's threshold for a dead chain: one number for both. + limit = chainStallTimeoutFromHorizon (chainForecastHorizon genesis) + slotLength = SL.fromNominalDiffTimeMicro (sgSlotLength genesis) + slotStart (SlotNo slot) = addUTCTime (fromIntegral slot * slotLength) (sgSystemStart genesis) + +-- | Check the status of one node in the testnet. +probeNode + :: FilePath -- ^ output directory + -> Nat -- ^ timeout, in seconds + -> Word32 -- ^ network magic + -> NominalDiffTime -- ^ tip age limit + -> (SlotNo -> UTCTime) -- ^ start time of a slot + -> ManifestNode + -> IO NodeProbeResult +probeNode outputDir timeoutSeconds magic limit slotStart + ManifestNode + { mnodeName = nodeName + , mnodeRole = role + , mnodePid = pid + , mnodeSocketPath = socketPath + } = do + answer <- askTip + pidAlive <- maybe (pure Nothing) isProcessAlive pid + pure NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = role + , probePid = pid + , probePidIsAlive = pidAlive + , probeNodeState = calcNodeState limit pidAlive answer + , probeNodeTipInfo = case answer of + AnsweredTip tip -> Just tip + AnsweredNoBlocks -> Nothing + NoAnswer _ -> Nothing + , probeError = case answer of + NoAnswer reason -> Just reason + _ -> Nothing + } + where + askTip :: IO NodeAnswer + askTip = do + -- Any failure (no socket, refused, handshake error) counts as no answer. + result <- tryAny $ timeout (fromIntegral (timeoutSeconds * 1_000_000)) (getLocalChainTip connectInfo) + answeredAt <- getCurrentTime + pure $ case result of + Right (Just (ChainTip slot hash blockNo)) -> + AnsweredTip TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipHash = hash + , tipAge = answeredAt `diffUTCTime` slotStart slot + } + Right (Just ChainTipAtGenesis) -> AnsweredNoBlocks + Right Nothing -> NoAnswer $ "no answer within " <> show timeoutSeconds <> "s" + Left e -> NoAnswer $ displayException e + + connectInfo :: LocalNodeConnectInfo + connectInfo = LocalNodeConnectInfo + { localNodeSocketPath = mapFile (outputDir ) socketPath + , localNodeNetworkId = Testnet (NetworkMagic $ fromIntegral magic) + , localConsensusModeParams = CardanoModeParams testnetEpochSlots + } + +-- | The rule for the whole network. +classifyNetwork :: NonEmpty NodeState -> NetworkStatus +classifyNetwork states + | all (== NodeOk) states = NetworkRunning + | NodeOk `elem` states = NetworkDegraded + | NodeStalled `elem` states = NetworkStalled + | otherwise = NetworkStopped + +-- | The rule for one node. The answer decides; the pid only tells "down" +-- (the process is gone) from "unreachable" (alive, or unknown). +calcNodeState + :: NominalDiffTime -- ^ tip age limit + -> Maybe Bool -- ^ whether the node's process is alive, if known + -> NodeAnswer + -> NodeState +calcNodeState limit _pidAlive (AnsweredTip tip) + | tipAge tip <= limit = NodeOk + | otherwise = NodeStalled +calcNodeState _limit _pidAlive AnsweredNoBlocks = NodeStalled +calcNodeState _limit pidAlive (NoAnswer _) + | pidAlive == Just False = NodeDown + | otherwise = NodeUnreachable + +-- | The rule for the best tip: the newest one, which is the one with the +-- highest slot. The ages cannot tell: each age is measured when its node +-- answered, and the nodes answer at different times. +newestTip :: [TipInfo] -> Maybe TipInfo +newestTip = fmap (maximumBy (comparing tipSlot)) . nonEmpty + +-- | The 'StatusReport' when there is no manifest, with a hint about why. +noTestnetReport :: FilePath -> IO StatusReport +noTestnetReport outputDir = do + now <- getCurrentTime + dirExists <- doesDirectoryExist outputDir + hasGenesis <- doesFileExist $ outputDir defaultGenesisFilepath ShelleyEra + let explanation + | not dirExists = DirMissing + | hasGenesis = NotReady + | otherwise = NotATestnetDir + pure StatusReport + { reportOutputDir = outputDir + , reportCheckedAt = now + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just explanation + } + +-- | Render a 'StatusReport' in a human readable way. +renderStatusReportHumanReadable :: StatusReport -> String +renderStatusReportHumanReadable + StatusReport + { reportStatus = status + , reportBestTip = bestTip + , reportNodes = nodes + , reportOutputDir = outputDir + , reportProblemExplanation = problemExplanation + } = + unlines $ ("Network: " <> summary status) : details <> errors + where + notOk = length $ filter ((/= NodeOk) . probeNodeState) nodes + + seconds :: NominalDiffTime -> String + seconds s = printf "%.1fs" (realToFrac s :: Double) + + summary :: NetworkStatus -> String + summary NetworkRunning = + "RUNNING - chain is producing blocks" + <> foldMap + ( \TipInfo + { tipSlot = slotNo + , tipBlockNo = blockNo + , tipAge = age + } -> " (tip: slot " + <> show (unSlotNo slotNo) + <> ", block " + <> show (unBlockNo blockNo) + <> ", " + <> seconds age + <> " old)" + ) + bestTip + summary NetworkDegraded = + "DEGRADED - chain is producing blocks, but " + <> show notOk + <> " of " + <> show (length nodes) + <> (if notOk == 1 then " nodes is" else " nodes are") + <> " not ok" + summary NetworkStalled = + "STALLED - chain is not producing blocks" + <> case bestTip of + Just t -> " (tip is " <> seconds (tipAge t) <> " old)" + _ -> " (no node has a block yet)" + summary NetworkStopped = "STOPPED - no node answers" + summary NoTestnet = "NO-TESTNET - no " <> defaultManifestFile <> " in " <> outputDir + + details :: [String] + details = + case status of + NoTestnet -> ["No testnet" <> maybe "" (\x -> " (" <> explain x <> ")") problemExplanation] + _ -> renderTable $ + ["NODE", "ROLE", "STATE", "PID", "TIP SLOT", "TIP BLOCK", "TIP AGE"] + : [ [ nodeName + , case nodeRole of + RoleSpo -> "spo" + RoleRelay -> "relay" + , case nodeState of + NodeOk -> "ok" + NodeStalled -> "stalled" + NodeUnreachable -> "unreachable" + NodeDown -> "down" + , maybe "-" show mNodePid + ] <> case mNodeTipInfo of + Nothing -> ["-", "-", "-"] + Just ( TipInfo + { tipSlot = slot + , tipBlockNo = blockNo + , tipAge = age + } + ) -> [ show $ unSlotNo slot + , show $ unBlockNo blockNo + , seconds age + ] + | NodeProbeResult + { probeNodeName = nodeName + , probeNodeRole = nodeRole + , probePid = mNodePid + , probeNodeState = nodeState + , probeNodeTipInfo = mNodeTipInfo + } + <- nodes + ] + + explain :: NoTestnetReason -> String + explain DirMissing = "The directory " <> outputDir <> " does not exist." + explain NotReady = "Genesis files exist, but no manifest: the testnet is still starting or it never became ready." + explain NotATestnetDir = outputDir <> " is not a testnet output directory." + + errors :: [String] + errors = + [ nodeName <> ": " <> err + | NodeProbeResult{probeNodeName = nodeName, probeError = Just err} <- nodes + ] + +-- | Render an array of strings as a table, where each string in the output list is a line. +renderTable :: [[String]] -> [String] +renderTable rows = + ("+" ++ concat ["-" ++ replicate width '-' ++ "-+" | width <- widths]) + : concatMap + ( \row -> + [ concat ("|" : [" " ++ pad width cell ++ " |" | (cell, width) <- zip row widths]) + , concat ("+" : ["-" ++ replicate width '-' ++ "-+" | width <- widths]) + ] + ) + rows + where + widths :: [Int] + widths = map (maximum . map length) (transpose rows) + + pad :: Int -> String -> [Char] + pad w s = s ++ replicate (w - length s) ' ' + diff --git a/cardano-testnet/src/Testnet/Status/Types.hs b/cardano-testnet/src/Testnet/Status/Types.hs new file mode 100644 index 00000000000..fc955fad342 --- /dev/null +++ b/cardano-testnet/src/Testnet/Status/Types.hs @@ -0,0 +1,182 @@ +{-# LANGUAGE InstanceSigs #-} +{-# LANGUAGE OverloadedStrings #-} + +module Testnet.Status.Types + ( CheckStatusOptions (..) + , NetworkStatus(..) + , NoTestnetReason(..) + , NodeAnswer(..) + , NodeProbeResult(..) + , NodeState(..) + , OutputFormat(..) + , StatusReport(..) + , TipInfo(..) + , exitCodeForStatus + ) where + +import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), ToJSON (..)) + +import Cardano.Prelude (ExitCode (..), Nat) + +import Prelude + +import Data.Aeson (Value, object, (.=)) +import Data.Default.Class (Default (..)) +import Data.Time (NominalDiffTime, UTCTime) +import System.Process (Pid) + +import Testnet.Manifest (ManifestNodeRole) + +-- Option types + +-- | Parameters for the `status` subcommand +data CheckStatusOptions = CheckStatusOptions + { testnetDir :: Maybe FilePath -- ^ the directory with the testnet environment, by default "./testnet" + , statusCheckTimeout :: Nat -- ^ the timeout for waiting for each node to respond to the status check in seconds + , outputFormat :: OutputFormat -- ^ the format in which to output the status information + } + +-- | Output format for the `status` subcommand +data OutputFormat = OutputFormatText | OutputFormatJson + +instance Default OutputFormat where + def :: OutputFormat + def = OutputFormatText + +-- Output types + +-- | Version of the JSON output, pinned by +-- @cardano-testnet/schemas/status.schema.json@. Any change to the output +-- shape, including new fields, bumps it. +statusSchemaVersion :: Int +statusSchemaVersion = 1 + +-- | The state of the whole network. +data NetworkStatus + = NetworkRunning -- ^ every node is 'NodeOk' + | NetworkDegraded -- ^ some nodes are 'NodeOk', but not all + | NetworkStalled -- ^ some nodes answer, but none is 'NodeOk' + | NetworkStopped -- ^ no node answers + | NoTestnet -- ^ there is no manifest file in the 'testnetDir': there is no + -- testnet, or it is still starting (the manifest is written + -- once the nodes are ready) + deriving (Eq, Show) + +instance ToJSON NetworkStatus where + toJSON :: NetworkStatus -> Value + toJSON NetworkRunning = "running" + toJSON NetworkDegraded = "degraded" + toJSON NetworkStalled = "stalled" + toJSON NetworkStopped = "stopped" + toJSON NoTestnet = "no-testnet" + +-- | Why there is no manifest file in the 'testnetDir'. +data NoTestnetReason + = DirMissing -- ^ the directory does not exist + | NotReady -- ^ it has genesis files: the testnet is still starting, or it never became ready + | NotATestnetDir -- ^ it is not the output directory of a testnet + deriving (Eq, Show) + +instance ToJSON NoTestnetReason where + toJSON :: NoTestnetReason -> Value + toJSON DirMissing = "dir-missing" + toJSON NotReady = "not-ready" + toJSON NotATestnetDir = "not-a-testnet-dir" + +-- | The state of one node. +data NodeState + = NodeOk -- ^ it answers, and its tip is fresh + | NodeStalled -- ^ it answers, but its tip is too old, or it has no blocks yet + | NodeUnreachable -- ^ it does not answer, but process may be alive + | NodeDown -- ^ it does not answer and the process is gone + deriving (Eq, Show) + +instance ToJSON NodeState where + toJSON :: NodeState -> Value + toJSON NodeOk = "ok" + toJSON NodeStalled = "stalled" + toJSON NodeUnreachable = "unreachable" + toJSON NodeDown = "down" + +-- | What a node said when it was asked for its chain tip. +data NodeAnswer + = AnsweredTip !TipInfo -- ^ it has blocks; this is the newest + | AnsweredNoBlocks -- ^ it answered, but it has no blocks yet + | NoAnswer !String -- ^ it refused, failed, or did not answer in time: why + deriving (Eq, Show) + +-- | A node's chain tip. +data TipInfo = TipInfo + { tipSlot :: !SlotNo + , tipBlockNo :: !BlockNo + , tipHash :: !(Hash BlockHeader) -- ^ hex + , tipAge :: !NominalDiffTime -- ^ time since the start of the tip's slot + } deriving (Eq, Show) + +instance ToJSON TipInfo where + toJSON :: TipInfo -> Value + toJSON t = object + [ "slot" .= unSlotNo (tipSlot t) + , "blockNo" .= unBlockNo (tipBlockNo t) + , "hash" .= tipHash t + , "ageSeconds" .= tipAge t + ] + +-- | What 'checkStatus' found out about one node. +data NodeProbeResult = NodeProbeResult + { probeNodeName :: !String + , probeNodeRole :: !ManifestNodeRole + , probePid :: !(Maybe Pid) + , probePidIsAlive :: !(Maybe Bool) -- ^ 'Nothing' when unknown: no pid, or Windows + , probeNodeState :: !NodeState + , probeNodeTipInfo :: !(Maybe TipInfo) -- ^ 'Just' when the node answered with a tip + , probeError :: !(Maybe String) -- ^ 'Just' why, when the node did not answer + } deriving (Eq, Show) + +instance ToJSON NodeProbeResult where + toJSON :: NodeProbeResult -> Value + toJSON n = object + [ "name" .= probeNodeName n + , "role" .= probeNodeRole n + , "pid" .= (fromIntegral <$> probePid n :: Maybe Int) + , "pidAlive" .= probePidIsAlive n + , "state" .= probeNodeState n + , "tip" .= probeNodeTipInfo n + , "error" .= probeError n + ] + +-- | Output of status command +data StatusReport = StatusReport + { reportOutputDir :: !FilePath + , reportCheckedAt :: !UTCTime + , reportStatus :: !NetworkStatus + , reportBestTip :: !(Maybe TipInfo) -- ^ the newest tip of all nodes: the one with the highest slot + , reportNodes :: ![NodeProbeResult] -- ^ empty for 'NoTestnet' + , reportProblemExplanation :: !(Maybe NoTestnetReason) -- ^ why there is no testnet + } deriving (Eq, Show) + +instance ToJSON StatusReport where + toJSON :: StatusReport -> Value + toJSON r = object + [ "schemaVersion" .= statusSchemaVersion + , "checkedAt" .= reportCheckedAt r + , "status" .= reportStatus r + , "exitCode" .= exitCodeNumber (exitCodeForStatus (reportStatus r)) + , "chainProducingBlocks" .= (exitCodeForStatus (reportStatus r) == ExitSuccess) + , "bestTip" .= reportBestTip r + , "nodes" .= reportNodes r + , "noTestnetReason" .= reportProblemExplanation r + ] + where + -- Extract code number from 'ExitCode' + exitCodeNumber :: ExitCode -> Int + exitCodeNumber ExitSuccess = 0 + exitCodeNumber (ExitFailure n) = n + +-- | Convert the network state to an exit code +exitCodeForStatus :: NetworkStatus -> ExitCode +exitCodeForStatus NetworkRunning = ExitSuccess +exitCodeForStatus NetworkDegraded = ExitSuccess +exitCodeForStatus NetworkStalled = ExitFailure 3 +exitCodeForStatus NetworkStopped = ExitFailure 4 +exitCodeForStatus NoTestnet = ExitFailure 5 diff --git a/cardano-testnet/src/Testnet/Types.hs b/cardano-testnet/src/Testnet/Types.hs index 5cb0cd703b6..7170c4509b4 100644 --- a/cardano-testnet/src/Testnet/Types.hs +++ b/cardano-testnet/src/Testnet/Types.hs @@ -26,6 +26,7 @@ module Testnet.Types , nodeGrpcServer , nodeConnectionInfo , testnetNodeConnectionInfo + , testnetEpochSlots , isTestnetNodeSpo , SpoNodeKeys(..) , Delegator(..) @@ -222,9 +223,14 @@ testnetNodeConnectionInfo testnetMagic node = LocalNodeConnectInfo { localNodeSocketPath= nodeSocketPath node , localNodeNetworkId=Testnet (NetworkMagic $ fromIntegral testnetMagic) - , localConsensusModeParams=CardanoModeParams $ EpochSlots 21600} + , localConsensusModeParams=CardanoModeParams testnetEpochSlots } +-- | The Byron epoch size used when connecting to a testnet node. Shared with +-- @cardano-testnet status@, so that both connect in the same way. +testnetEpochSlots :: EpochSlots +testnetEpochSlots = EpochSlots 21600 + data SpoNodeKeys = SpoNodeKeys { poolNodeKeysCold :: KeyPair StakePoolKey , poolNodeKeysVrf :: KeyPair VrfKey diff --git a/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs new file mode 100644 index 00000000000..a7c393ca52f --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/Cardano/Testnet/Test/Golden/Status.hs @@ -0,0 +1,203 @@ +{-# LANGUAGE OverloadedStrings #-} + +{- HLINT ignore "Use camelCase" -} + +module Cardano.Testnet.Test.Golden.Status + ( golden_StatusJson + , golden_StatusTable + , golden_StatusCommand + ) where + +import Cardano.Api (BlockHeader, BlockNo (..), Hash, SlotNo (..), + deserialiseFromRawBytesHex) + +import Cardano.Testnet.Test.Golden.Util (execDetailCardanoTestnet) + +import Prelude + +import Data.Aeson (Value (..), eitherDecodeStrict) +import Data.Aeson.Encode.Pretty (encodePretty) +import qualified Data.Aeson.KeyMap as KeyMap +import Data.ByteString (ByteString) +import qualified Data.ByteString.Lazy as LBS +import Data.List (isInfixOf) +import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text +import Data.Time.Calendar (fromGregorian) +import Data.Time.Clock (UTCTime (..)) +import System.Exit (ExitCode (..)) +import System.FilePath (()) + +import Testnet.Manifest (ManifestNodeRole (..)) +import Testnet.Status.Run (renderStatusReportHumanReadable) +import Testnet.Status.Types + +import Hedgehog (Property, (===)) +import Hedgehog.Extras.Test.Base (assertWith, leftFail, moduleWorkspace, propertyOnce) +import qualified Hedgehog.Extras.Test.File as H +import Hedgehog.Extras.Test.Golden (diffVsGoldenFile) +import qualified Hedgehog.Extras.Test.Process as H + +-- | A fixed sample report with one node in each state, so that every node +-- field, with and without a value, is pinned. +sampleReport :: StatusReport +sampleReport = StatusReport + { reportOutputDir = "testnet" + , reportCheckedAt = UTCTime (fromGregorian 2026 1 1) 60 + , reportStatus = NetworkDegraded + , reportBestTip = Just freshTip + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled (Just oldTip) Nothing + , NodeProbeResult "node3" RoleRelay Nothing Nothing NodeUnreachable Nothing + (Just "no answer within 5s") + , NodeProbeResult "node4" RoleRelay (Just 12348) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") + ] + , reportProblemExplanation = Nothing + } + +-- | The tips of the sample reports: a fresh one and an old one. +freshTip, oldTip :: TipInfo +freshTip = TipInfo (SlotNo 1234) (BlockNo 56) + (sampleHash "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a") 0.5 +oldTip = TipInfo (SlotNo 400) (BlockNo 20) + (sampleHash "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5") 83.4 + +-- | Fixed sample reports for the other network states, for the text goldens. +sampleRunningReport :: StatusReport +sampleRunningReport = sampleReport + { reportStatus = NetworkRunning + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeOk (Just freshTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeOk (Just freshTip) Nothing + ] + } + +sampleStalledReport :: StatusReport +sampleStalledReport = sampleReport + { reportStatus = NetworkStalled + , reportBestTip = Just oldTip + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeStalled (Just oldTip) Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") + ] + } + +sampleStalledNoBlocksReport :: StatusReport +sampleStalledNoBlocksReport = sampleReport + { reportStatus = NetworkStalled + , reportBestTip = Nothing + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just True) NodeStalled Nothing Nothing + , NodeProbeResult "node2" RoleRelay (Just 12346) (Just True) NodeStalled Nothing Nothing + ] + } + +sampleStoppedReport :: StatusReport +sampleStoppedReport = sampleReport + { reportStatus = NetworkStopped + , reportBestTip = Nothing + , reportNodes = + [ NodeProbeResult "node1" RoleSpo (Just 12345) (Just False) NodeDown Nothing + (Just "Network.Socket.connect: : does not exist (Connection refused)") + , NodeProbeResult "node2" RoleRelay Nothing Nothing NodeUnreachable Nothing + (Just "no answer within 5s") + ] + } + +-- | A fixed sample report for a directory without a testnet: it has no nodes +-- and no tip, so the empty and null forms of the output are pinned. +sampleNoTestnetReport :: StatusReport +sampleNoTestnetReport = StatusReport + { reportOutputDir = "testnet" + , reportCheckedAt = UTCTime (fromGregorian 2026 1 1) 60 + , reportStatus = NoTestnet + , reportBestTip = Nothing + , reportNodes = [] + , reportProblemExplanation = Just DirMissing + } + +-- | Parse a fixture hash, failing loudly on a typo in the literal. +sampleHash :: ByteString -> Hash BlockHeader +sampleHash h = + either (\e -> error $ "sampleReport: bad fixture hash: " <> show e) id $ + deserialiseFromRawBytesHex h + +goldenFile :: FilePath -> FilePath +goldenFile name = "cardano-testnet/test/cardano-testnet-golden/files/golden" name + +-- | The golden files are also checked against +-- @cardano-testnet/schemas/status.schema.json@ (@make testnet-status-validate@), +-- so together they check that the output matches the schema. +-- +-- Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusJson/"'@ +golden_StatusJson :: Property +golden_StatusJson = propertyOnce $ do + base <- H.getProjectBase + diffVsGoldenFile (encode sampleReport) $ base goldenFile "status.json" + diffVsGoldenFile (encode sampleNoTestnetReport) $ base goldenFile "status_no_testnet.json" + where + encode = Text.unpack . Text.decodeUtf8 . LBS.toStrict . encodePretty + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusTable/"'@ +golden_StatusTable :: Property +golden_StatusTable = propertyOnce $ do + base <- H.getProjectBase + diffVsGoldenFile (renderStatusReportHumanReadable sampleReport) $ + base goldenFile "status.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleNoTestnetReport) $ + base goldenFile "status_no_testnet.txt" + diffVsGoldenFile + (renderStatusReportHumanReadable sampleNoTestnetReport{reportProblemExplanation = Just NotReady}) $ + base goldenFile "status_no_testnet_not_ready.txt" + diffVsGoldenFile + (renderStatusReportHumanReadable sampleNoTestnetReport{reportProblemExplanation = Just NotATestnetDir}) $ + base goldenFile "status_no_testnet_not_a_testnet_dir.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleRunningReport) $ + base goldenFile "status_running.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStalledReport) $ + base goldenFile "status_stalled.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStalledNoBlocksReport) $ + base goldenFile "status_stalled_no_blocks.txt" + diffVsGoldenFile (renderStatusReportHumanReadable sampleStoppedReport) $ + base goldenFile "status_stopped.txt" + +-- | The real command, without a testnet: the exit code, stdout and stderr for +-- a missing directory, as a table and as JSON, and for a broken manifest. The +-- JSON is the golden one, apart from the time of the check. +-- +-- Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-golden --test-options '-p "/golden_StatusCommand/"'@ +golden_StatusCommand :: Property +golden_StatusCommand = propertyOnce . moduleWorkspace "status-command" $ \dir -> do + base <- H.getProjectBase + let missing = dir "missing" + + (code, out, err) <- execDetailCardanoTestnet ["status", "--testnet-dir", missing] + code === ExitFailure 5 + lines (filter (/= '\r') out) === + [ "Network: NO-TESTNET - no manifest.json in " <> missing + , "No testnet (The directory " <> missing <> " does not exist.)" + ] + err === "" + + (jsonCode, jsonOut, jsonErr) <- execDetailCardanoTestnet ["status", "--testnet-dir", missing, "--json"] + jsonCode === ExitFailure 5 + json <- leftFail (eitherDecodeStrict (Text.encodeUtf8 $ Text.pack jsonOut) :: Either String Value) + golden <- H.readJsonFileOk $ base goldenFile "status_no_testnet.json" + withoutCheckedAt json === withoutCheckedAt golden + jsonErr === "" + + H.writeFile (dir "manifest.json") "{" + (brokenCode, brokenOut, brokenErr) <- execDetailCardanoTestnet ["status", "--testnet-dir", dir] + brokenCode === ExitFailure 1 + brokenOut === "" + assertWith brokenErr ("manifest.json" `isInfixOf`) + where + withoutCheckedAt :: Value -> Value + withoutCheckedAt (Object o) = Object (KeyMap.delete "checkedAt" o) + withoutCheckedAt v = v diff --git a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs index 34f78d90659..e52012aab56 100644 --- a/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs +++ b/cardano-testnet/test/cardano-testnet-golden/cardano-testnet-golden.hs @@ -8,6 +8,7 @@ import qualified Cardano.Crypto.Init as Crypto import qualified Cardano.Testnet.Test.Golden.Config import qualified Cardano.Testnet.Test.Golden.Help import qualified Cardano.Testnet.Test.Golden.Manifest +import qualified Cardano.Testnet.Test.Golden.Status import Prelude @@ -30,6 +31,9 @@ tests = pure $ T.testGroup "Golden tests" , ignoreOnWindows "golden_HelpReadme" Cardano.Testnet.Test.Golden.Help.golden_HelpReadme , H.testPropertyNamed "golden_VersionCmd" (fromString "golden_VersionCmd") Cardano.Testnet.Test.Golden.Help.golden_VersionCmd , H.testPropertyNamed "golden_Manifest" (fromString "golden_Manifest") Cardano.Testnet.Test.Golden.Manifest.golden_Manifest + , H.testPropertyNamed "golden_StatusJson" (fromString "golden_StatusJson") Cardano.Testnet.Test.Golden.Status.golden_StatusJson + , H.testPropertyNamed "golden_StatusTable" (fromString "golden_StatusTable") Cardano.Testnet.Test.Golden.Status.golden_StatusTable + , H.testPropertyNamed "golden_StatusCommand" (fromString "golden_StatusCommand") Cardano.Testnet.Test.Golden.Status.golden_StatusCommand ] ingredients :: [T.Ingredient] diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli index 12a97b9e81b..1a34d1930e0 100644 --- a/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help.cli @@ -1,4 +1,4 @@ -Usage: cardano-testnet (cardano | create-env | version | help) +Usage: cardano-testnet (cardano | create-env | status | version | help) Usage: cardano-testnet cardano [ --node-env FILEPATH [--preserve-timestamps] @@ -39,6 +39,13 @@ Usage: cardano-testnet create-env Create a sandbox for Cardano testnet +Usage: cardano-testnet status [--testnet-dir FILEPATH] + [--timeout SECONDS] + [--json] + + + Check the status of a running testnet + Usage: cardano-testnet version Show cardano-testnet version diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli new file mode 100644 index 00000000000..b8a19886b61 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/help/status.cli @@ -0,0 +1,14 @@ +Usage: cardano-testnet status [--testnet-dir FILEPATH] + [--timeout SECONDS] + [--json] + + + Check the status of a running testnet + +Available options: + --testnet-dir FILEPATH Path to the environment folder of the testnet whose + status to check, by default "./testnet". + --timeout SECONDS How many seconds to wait for each node to answer (1 - + 3600). (default: 5) + --json Output status as JSON instead of a table. + -h,--help Show this help text diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json new file mode 100644 index 00000000000..7d8b35e03f9 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.json @@ -0,0 +1,62 @@ +{ + "bestTip": { + "ageSeconds": 0.5, + "blockNo": 56, + "hash": "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a", + "slot": 1234 + }, + "chainProducingBlocks": true, + "checkedAt": "2026-01-01T00:01:00Z", + "exitCode": 0, + "noTestnetReason": null, + "nodes": [ + { + "error": null, + "name": "node1", + "pid": 12345, + "pidAlive": true, + "role": "spo", + "state": "ok", + "tip": { + "ageSeconds": 0.5, + "blockNo": 56, + "hash": "cc061706c16ad2b35450f1528f5652ff172f9b0acabcdc29284e2363d3bebb2a", + "slot": 1234 + } + }, + { + "error": null, + "name": "node2", + "pid": 12346, + "pidAlive": true, + "role": "relay", + "state": "stalled", + "tip": { + "ageSeconds": 83.4, + "blockNo": 20, + "hash": "5b1e29bbc6e3e4d1f4f4f6bb2dbd3c1b2a7c68b7a8a4b3a1c4e0d2f9e8a7b6c5", + "slot": 400 + } + }, + { + "error": "no answer within 5s", + "name": "node3", + "pid": null, + "pidAlive": null, + "role": "relay", + "state": "unreachable", + "tip": null + }, + { + "error": "Network.Socket.connect: : does not exist (Connection refused)", + "name": "node4", + "pid": 12348, + "pidAlive": false, + "role": "relay", + "state": "down", + "tip": null + } + ], + "schemaVersion": 1, + "status": "degraded" +} \ No newline at end of file diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt new file mode 100644 index 00000000000..66d9a286838 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt @@ -0,0 +1,14 @@ +Network: DEGRADED - chain is producing blocks, but 3 of 4 nodes are not ok ++-------+-------+-------------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node1 | spo | ok | 12345 | 1234 | 56 | 0.5s | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node2 | relay | stalled | 12346 | 400 | 20 | 83.4s | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node3 | relay | unreachable | - | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node4 | relay | down | 12348 | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +node3: no answer within 5s +node4: Network.Socket.connect: : does not exist (Connection refused) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json new file mode 100644 index 00000000000..94a7a2b4db9 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json @@ -0,0 +1,10 @@ +{ + "bestTip": null, + "chainProducingBlocks": false, + "checkedAt": "2026-01-01T00:01:00Z", + "exitCode": 5, + "noTestnetReason": "dir-missing", + "nodes": [], + "schemaVersion": 1, + "status": "no-testnet" +} \ No newline at end of file diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt new file mode 100644 index 00000000000..58e7f5b84ee --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (The directory testnet does not exist.) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt new file mode 100644 index 00000000000..dd6c9a75b33 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (testnet is not a testnet output directory.) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt new file mode 100644 index 00000000000..0b1d8446cb9 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt @@ -0,0 +1,2 @@ +Network: NO-TESTNET - no manifest.json in testnet +No testnet (Genesis files exist, but no manifest: the testnet is still starting or it never became ready.) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt new file mode 100644 index 00000000000..c10d62d5365 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt @@ -0,0 +1,8 @@ +Network: RUNNING - chain is producing blocks (tip: slot 1234, block 56, 0.5s old) ++-------+-------+-------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------+-------+----------+-----------+---------+ +| node1 | spo | ok | 12345 | 1234 | 56 | 0.5s | ++-------+-------+-------+-------+----------+-----------+---------+ +| node2 | relay | ok | 12346 | 1234 | 56 | 0.5s | ++-------+-------+-------+-------+----------+-----------+---------+ diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt new file mode 100644 index 00000000000..b0fbd1af712 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt @@ -0,0 +1,9 @@ +Network: STALLED - chain is not producing blocks (tip is 83.4s old) ++-------+-------+---------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+---------+-------+----------+-----------+---------+ +| node1 | spo | stalled | 12345 | 400 | 20 | 83.4s | ++-------+-------+---------+-------+----------+-----------+---------+ +| node2 | relay | down | 12346 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ +node2: Network.Socket.connect: : does not exist (Connection refused) diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt new file mode 100644 index 00000000000..8e2a4e16f11 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt @@ -0,0 +1,8 @@ +Network: STALLED - chain is not producing blocks (no node has a block yet) ++-------+-------+---------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+---------+-------+----------+-----------+---------+ +| node1 | spo | stalled | 12345 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ +| node2 | relay | stalled | 12346 | - | - | - | ++-------+-------+---------+-------+----------+-----------+---------+ diff --git a/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt new file mode 100644 index 00000000000..425d128dab0 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt @@ -0,0 +1,10 @@ +Network: STOPPED - no node answers ++-------+-------+-------------+-------+----------+-----------+---------+ +| NODE | ROLE | STATE | PID | TIP SLOT | TIP BLOCK | TIP AGE | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node1 | spo | down | 12345 | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +| node2 | relay | unreachable | - | - | - | - | ++-------+-------+-------------+-------+----------+-----------+---------+ +node1: Network.Socket.connect: : does not exist (Connection refused) +node2: no answer within 5s diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs new file mode 100644 index 00000000000..b3c298082d2 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Status.hs @@ -0,0 +1,98 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.Testnet.Test.Status + ( hprop_status + , hprop_status_no_testnet + ) where + +import Cardano.Api (CardanoEra (..)) + +import Cardano.Testnet (TestnetNode (..), TestnetRuntime (..), createAndRunTestnet, + defaultManifestFile, isTestnetNodeSpo, mkConf) + +import Prelude + +import Control.Monad (forM_, void) +import Data.Default.Class (def) +import Data.Either (isLeft) +import Data.Foldable (toList) +import Data.List (find) +import Data.Maybe (isJust) +import System.FilePath (()) +import qualified System.Process as Process + +import Testnet.Defaults (defaultGenesisFilepath) +import Testnet.Property.Util (integrationRetryWorkspace) +import Testnet.Signal (hardKillProcess) +import Testnet.Status.Run (checkStatus) +import Testnet.Status.Types + +import Hedgehog (Property, (===)) +import qualified Hedgehog as H +import qualified Hedgehog.Extras as H + +import RIO (tryAny) + +-- | Integration test: a fresh testnet is RUNNING, with a best tip. After one +-- relay is killed, it is DEGRADED, and the killed relay is down. After the +-- other nodes are killed too, it is STOPPED, and every node is down. +-- +-- Execute with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status reports network health/"'@ +hprop_status :: Property +hprop_status = integrationRetryWorkspace 2 "status" $ \tmpDir -> H.runWithDefaultWatchdog_ $ do + conf <- mkConf tmpDir + TestnetRuntime{testnetNodes} <- createAndRunTestnet def def conf + + running <- H.evalIO $ checkStatus tmpDir 5 + reportStatus running === NetworkRunning + map probeNodeState (reportNodes running) === map (const NodeOk) (toList testnetNodes) + H.assertWith (reportBestTip running) isJust + + relay <- H.evalMaybe $ find (not . isTestnetNodeSpo) testnetNodes + H.evalIO $ do + hardKillProcess (nodeProcessHandle relay) + -- Wait for the process, so that its pid is gone and status can tell "down". + void . Process.waitForProcess $ nodeProcessHandle relay + + degraded <- H.evalIO $ checkStatus tmpDir 5 + reportStatus degraded === NetworkDegraded + killed <- H.evalMaybe $ find ((== nodeName relay) . probeNodeName) (reportNodes degraded) + probeNodeState killed === NodeDown + H.assertWith (probeError killed) isJust + + H.evalIO . forM_ (filter ((/= nodeName relay) . nodeName) $ toList testnetNodes) $ \node -> do + hardKillProcess (nodeProcessHandle node) + void . Process.waitForProcess $ nodeProcessHandle node + + stopped <- H.evalIO $ checkStatus tmpDir 5 + reportStatus stopped === NetworkStopped + map probeNodeState (reportNodes stopped) === map (const NodeDown) (toList testnetNodes) + reportBestTip stopped === Nothing + +-- | 'checkStatus' without a running testnet: a directory without a manifest +-- is NO-TESTNET, with a hint about why, and a broken manifest is an error. +-- +-- Execute with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status without a testnet/"'@ +hprop_status_no_testnet :: Property +hprop_status_no_testnet = H.propertyOnce . H.workspace "status-no-testnet" $ \dir -> do + let missingDir = dir "missing" + missing <- H.evalIO $ checkStatus missingDir 1 + reportStatus missing === NoTestnet + reportProblemExplanation missing === Just DirMissing + + notTestnet <- H.evalIO $ checkStatus dir 1 + reportStatus notTestnet === NoTestnet + reportProblemExplanation notTestnet === Just NotATestnetDir + + -- The genesis files are written before the manifest, while the testnet starts. + H.writeFile (dir defaultGenesisFilepath ShelleyEra) "{}" + starting <- H.evalIO $ checkStatus dir 1 + reportStatus starting === NoTestnet + reportProblemExplanation starting === Just NotReady + + H.writeFile (dir defaultManifestFile) "{" + broken <- H.evalIO . tryAny $ checkStatus dir 1 + H.assertWith broken isLeft diff --git a/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs new file mode 100644 index 00000000000..11eaedaa491 --- /dev/null +++ b/cardano-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/StatusRules.hs @@ -0,0 +1,81 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Cardano.Testnet.Test.StatusRules + ( unit_node_state + , unit_classify_network + , unit_exit_codes + , unit_best_tip + ) where + +import Cardano.Api (BlockNo (..), SlotNo (..), deserialiseFromRawBytesHex) + +import Prelude + +import Data.List.NonEmpty (NonEmpty (..)) +import System.Exit (ExitCode (..)) + +import Testnet.Status.Run (calcNodeState, classifyNetwork, newestTip) +import Testnet.Status.Types + +import Hedgehog ((===)) +import qualified Hedgehog as H +import qualified Hedgehog.Extras as H + +-- | A tip of the given age, in seconds. +tipOfAge :: Double -> TipInfo +tipOfAge age = TipInfo (SlotNo 100) (BlockNo 10) sampleHash (realToFrac age) + where + sampleHash = + either (\e -> error $ "tipOfAge: bad fixture hash: " <> show e) id $ + deserialiseFromRawBytesHex "3525888dc2c18eb5f5f34cb0edf911aa8b91ba485d195bb7d70dff7a7a32af91" + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Node state/"'@ +unit_node_state :: H.Property +unit_node_state = H.propertyOnce $ do + let limit = 60 + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 1)) === NodeOk + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 60)) === NodeOk + calcNodeState limit (Just True) (AnsweredTip (tipOfAge 61)) === NodeStalled + -- A node that answers but has no blocks yet is stalled. + calcNodeState limit (Just True) AnsweredNoBlocks === NodeStalled + -- The answer decides: a node that answers is never down, whatever its pid says. + calcNodeState limit (Just False) (AnsweredTip (tipOfAge 1)) === NodeOk + -- No answer: the pid tells down from unreachable. + calcNodeState limit (Just False) (NoAnswer "refused") === NodeDown + calcNodeState limit (Just True) (NoAnswer "refused") === NodeUnreachable + calcNodeState limit Nothing (NoAnswer "refused") === NodeUnreachable + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Network status/"'@ +unit_classify_network :: H.Property +unit_classify_network = H.propertyOnce $ do + classifyNetwork (NodeOk :| [NodeOk, NodeOk]) === NetworkRunning + classifyNetwork (NodeOk :| [NodeDown, NodeUnreachable]) === NetworkDegraded + classifyNetwork (NodeOk :| [NodeStalled]) === NetworkDegraded + classifyNetwork (NodeStalled :| [NodeDown, NodeUnreachable]) === NetworkStalled + classifyNetwork (NodeDown :| [NodeUnreachable]) === NetworkStopped + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Status exit codes/"'@ +unit_exit_codes :: H.Property +unit_exit_codes = H.propertyOnce $ do + -- 0 means the chain is producing blocks, even when some nodes are not ok. + exitCodeForStatus NetworkRunning === ExitSuccess + exitCodeForStatus NetworkDegraded === ExitSuccess + exitCodeForStatus NetworkStalled === ExitFailure 3 + exitCodeForStatus NetworkStopped === ExitFailure 4 + exitCodeForStatus NoTestnet === ExitFailure 5 + +-- | Execute me with: +-- @DISABLE_RETRIES=1 cabal test cardano-testnet-test --test-options '-p "/Best tip/"'@ +unit_best_tip :: H.Property +unit_best_tip = H.propertyOnce $ do + -- With 0.1 s slots, node A answers at 10.2 s with slot 100, so its tip is + -- 0.2 s old. Node B answers later, at 12 s, with slot 110, so its tip is + -- 1 s old. B's tip is the newest, even though its age is bigger. + let tipA = (tipOfAge 0.2) { tipSlot = SlotNo 100 } + tipB = (tipOfAge 1) { tipSlot = SlotNo 110 } + newestTip [tipA, tipB] === Just tipB + newestTip [tipB, tipA] === Just tipB + newestTip [] === Nothing diff --git a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs index 955728500db..fbce4cf9d3b 100644 --- a/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs +++ b/cardano-testnet/test/cardano-testnet-test/cardano-testnet-test.hs @@ -48,6 +48,8 @@ import qualified Cardano.Testnet.Test.Rpc.Transaction import qualified Cardano.Testnet.Test.RunTestnet import qualified Cardano.Testnet.Test.SanityCheck import qualified Cardano.Testnet.Test.SanityCheck as LedgerEvents +import qualified Cardano.Testnet.Test.Status +import qualified Cardano.Testnet.Test.StatusRules import qualified Cardano.Testnet.Test.SubmitApi.Transaction import qualified Cardano.Testnet.Test.UpdateTimeStamps @@ -154,6 +156,9 @@ tests = do , ignoreOnWindows "Stale manifest is removed by createTestnetEnv" Cardano.Testnet.Test.Manifest.hprop_manifest_stale , H.testPropertyNamed "Manifest Windows pipe path" (fromString "hprop_manifest_windows_pipe_path") Cardano.Testnet.Test.Manifest.hprop_manifest_windows_pipe_path + , ignoreOnWindows "Status reports network health" Cardano.Testnet.Test.Status.hprop_status + , H.testPropertyNamed "Status without a testnet" (fromString "hprop_status_no_testnet") + Cardano.Testnet.Test.Status.hprop_status_no_testnet ] , T.testGroup "SubmitApi" [ ignoreOnMacAndWindows "transaction" Cardano.Testnet.Test.SubmitApi.Transaction.hprop_transaction @@ -168,6 +173,16 @@ tests = do , ignoreOnWindows "RPC Transaction Submit" Cardano.Testnet.Test.Rpc.Transaction.hprop_rpc_transaction , ignoreOnWindows "RPC Eval Tx" Cardano.Testnet.Test.Rpc.Eval.hprop_rpc_eval_tx ] + , T.testGroup "Status rules" + [ H.testPropertyNamed "Node state" (fromString "unit_node_state") + Cardano.Testnet.Test.StatusRules.unit_node_state + , H.testPropertyNamed "Network status" (fromString "unit_classify_network") + Cardano.Testnet.Test.StatusRules.unit_classify_network + , H.testPropertyNamed "Status exit codes" (fromString "unit_exit_codes") + Cardano.Testnet.Test.StatusRules.unit_exit_codes + , H.testPropertyNamed "Best tip" (fromString "unit_best_tip") + Cardano.Testnet.Test.StatusRules.unit_best_tip + ] , T.testGroup "NodesWithOptions parser" [ H.testPropertyNamed "Roundtrip" (fromString "prop_parseNodeSpecs_roundtrip") Cardano.Testnet.Test.Parser.prop_parseNodeSpecs_roundtrip diff --git a/nix/haskell.nix b/nix/haskell.nix index 470fcca7fd9..b9b09cf536b 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -285,6 +285,16 @@ let "configuration/defaults/byron-mainnet" "cardano-testnet/test/cardano-testnet-golden/files/golden/manifest.json" "cardano-testnet/test/cardano-testnet-golden/files/golden/node_default_config.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.json" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_a_testnet_dir.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_no_testnet_not_ready.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_running.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stalled_no_blocks.txt" + "cardano-testnet/test/cardano-testnet-golden/files/golden/status_stopped.txt" "cardano-testnet/test/cardano-testnet-test/files/golden/tx.failed.response.json.golden" "cardano-testnet/test/cardano-testnet-test/files/input/sample-constitution.txt" "cardano-testnet/files/data/alonzo/genesis.alonzo.spec.json"