diff --git a/cf-reactor/Makefile.am b/cf-reactor/Makefile.am index 3c84df15546..b13357ad59f 100644 --- a/cf-reactor/Makefile.am +++ b/cf-reactor/Makefile.am @@ -47,7 +47,10 @@ libcf_reactor_la_SOURCES = \ stoppable_thread.c stoppable_thread.h \ user_watcher.c user_watcher.h \ wakeup_channel.c wakeup_channel.h \ - watcher.c watcher.h + watch_backends.c watch_backends.h \ + watch_location.c watch_location.h \ + watcher.c watcher.h \ + watcher_report.c watcher_report.h if !BUILTIN_EXTENSIONS bin_PROGRAMS = cf-reactor diff --git a/cf-reactor/file_watcher.c b/cf-reactor/file_watcher.c index fa4f6d54809..352cd058cc2 100644 --- a/cf-reactor/file_watcher.c +++ b/cf-reactor/file_watcher.c @@ -26,12 +26,14 @@ #include #include #include -#include /* safe_fopen() */ +#include /* safe_open() */ #include /* HashDigestFromId() */ #include /* IntFromString() */ #include /* cf_strtimestamp_local() */ #include +#include #include +#include /* strerror() */ #include /* Minimum time between two logs of a persisting failure to check the file, @@ -40,6 +42,16 @@ #define CONTENT_HASH_METHOD HASH_METHOD_SHA256 +/* What changes when a file's content may have changed, see StatSignatureOf() */ +typedef struct +{ + dev_t dev; + ino_t ino; + off_t size; + struct timespec mtime; + struct timespec ctime; +} StatSignature; + typedef struct { char *path; @@ -60,6 +72,7 @@ typedef struct { bool hashed; /* Whether it was a regular file, that was hashed */ unsigned char digest[EVP_MAX_MD_SIZE + 1]; + StatSignature signature; /* of the file when it was hashed */ } content; /* FileWatcherCheckFilePermissionsChanged() */ @@ -81,6 +94,33 @@ typedef struct } last; } FileWatcherState; +/** + * Copy of path in a form stat() accepts. On Windows, msvcrt's stat() fails + * with ENOENT for a path with a trailing separator, or with the \\?\ (or + * //?/) prefix, so a watcher with such a path would never see its file. + */ +static char *WatchPathCopy(const char *path) +{ + assert(path != NULL); + + char *copy = xstrdup(path); +#ifdef __MINGW32__ + if ((copy[0] == '\\' || copy[0] == '/') && (copy[1] == '\\' || copy[1] == '/') && + copy[2] == '?' && (copy[3] == '\\' || copy[3] == '/')) + { + memmove(copy, copy + 4, strlen(copy + 4) + 1); + } + + /* Keeps the separator of a drive's root, e.g. C:/ */ + size_t length = strlen(copy); + while (length > 1 && (copy[length - 1] == '\\' || copy[length - 1] == '/') && + !(length == 3 && copy[1] == ':')) + { + copy[--length] = '\0'; + } +#endif + return copy; +} void *FileWatcherStateNew(ARG_UNUSED const EvalContext *ctx, const Rlist *args) { @@ -88,7 +128,7 @@ void *FileWatcherStateNew(ARG_UNUSED const EvalContext *ctx, const Rlist *args) const char *path = RlistScalarValue(args); FileWatcherState *fws = (FileWatcherState *) xcalloc(1, sizeof(FileWatcherState)); - fws->path = xstrdup(path); + fws->path = WatchPathCopy(path); return fws; } @@ -96,11 +136,15 @@ void *FileWatcherStateNew(ARG_UNUSED const EvalContext *ctx, const Rlist *args) void *FileWatcherSizeStateNew(ARG_UNUSED const EvalContext *ctx, const Rlist *args) { assert(args != NULL && args->next != NULL); + if (args == NULL || args->next == NULL) + { + return NULL; + } const char *path = RlistScalarValue(args); const long size_threshold = IntFromString(RlistScalarValue(args->next)); FileWatcherState *fws = (FileWatcherState *) xcalloc(1, sizeof(FileWatcherState)); - fws->path = xstrdup(path); + fws->path = WatchPathCopy(path); fws->last.file_size.threshold = size_threshold; return fws; @@ -129,8 +173,10 @@ static void LogCheckError(FileWatcherState *fws, const char *op, int err) if (fws->last_error_logged == 0 || now - fws->last_error_logged >= CHECK_ERROR_LOG_INTERVAL_SECS) { + /* strerror(), as err is an errno value: GetErrorStrFromCode() + * takes a Windows error code on Windows */ Log(LOG_LEVEL_ERR, "Unable to %s '%s' while checking for file events: %s", - op, fws->path, GetErrorStrFromCode(err)); + op, fws->path, strerror(err)); fws->last_error_logged = now; } } @@ -155,6 +201,19 @@ static bool FileLstat(FileWatcherState *fws, bool *exists, struct stat *sb) return true; } +#ifdef __MINGW32__ + /* msvcrt's stat() also fails with ENOENT while the file's directory is + * open without sharing (FindFirstFile() gets a sharing violation), when + * the file is still there: then whether it changed is unknown */ + WIN32_FILE_ATTRIBUTE_DATA attributes; + if (errno == ENOENT && + GetFileAttributesExA(fws->path, GetFileExInfoStandard, &attributes)) + { + LogCheckError(fws, "lstat", EBUSY); + return false; + } +#endif + if (errno == ENOENT || errno == ENOTDIR) { *exists = false; @@ -280,15 +339,103 @@ WatcherCheckResult FileWatcherCheckFileModified(void *state) // =========== code for the file_content_modified event =========== +static struct timespec StatCtime(const struct stat *sb) +{ + assert(sb != NULL); + +#if defined(HAVE_STRUCT_STAT_ST_MTIM) + return sb->st_ctim; +#elif defined(HAVE_STRUCT_STAT_ST_MTIMESPEC) + return sb->st_ctimespec; +#else + return (struct timespec) { .tv_sec = sb->st_ctime, .tv_nsec = 0 }; +#endif +} + +/** Content can't change without one of these changing too: writing updates + * mtime and ctime, replacing the file changes the inode, and setting mtime + * back still updates ctime. So the file is only hashed again when they + * differ, which keeps frequent checks of many files cheap. */ +static StatSignature StatSignatureOf(const struct stat *sb) +{ + assert(sb != NULL); + + StatSignature signature; + memset(&signature, 0, sizeof(signature)); /* no padding garbage for memcmp() */ + signature.dev = sb->st_dev; + signature.ino = sb->st_ino; + signature.size = sb->st_size; + signature.mtime = StatMtime(sb); + signature.ctime = StatCtime(sb); + return signature; +} + +/** Opens the regular file at path for reading. Fails with EINVAL if it isn't + * one after all: the caller lstat()ed it, but it may have been swapped for a + * FIFO since, and opening that for reading would block forever. */ +static int OpenRegularFile(const char *path) +{ + assert(path != NULL); + + int flags = O_RDONLY; +#ifdef O_NONBLOCK + flags |= O_NONBLOCK; +#endif +#ifdef O_NOFOLLOW + flags |= O_NOFOLLOW; +#endif +#ifdef O_BINARY + flags |= O_BINARY; +#endif + const int fd = safe_open(path, flags); + if (fd < 0) + { + return -1; + } + + struct stat sb; + if (fstat(fd, &sb) != 0) + { + const int err = errno; + close(fd); + errno = err; + return -1; + } + if (!S_ISREG(sb.st_mode)) + { + close(fd); + errno = EINVAL; + return -1; + } + +#ifndef __MINGW32__ + /* Regular files never block, so this is only to leave the fd as expected */ + const int fd_flags = fcntl(fd, F_GETFL); + if (fd_flags != -1) + { + fcntl(fd, F_SETFL, fd_flags & ~O_NONBLOCK); + } +#endif + return fd; +} + /* Hashes the content of the file into digest. Returns false and sets errno * on failure. */ static bool HashFileContent(const char *path, unsigned char digest[EVP_MAX_MD_SIZE + 1]) { assert(path != NULL); - FILE *file = safe_fopen(path, "rb"); + const int fd = OpenRegularFile(path); + if (fd < 0) + { + return false; + } + FILE *file = fdopen(fd, "rb"); if (file == NULL) { + const int err = errno; + close(fd); + errno = err; return false; } @@ -335,7 +482,15 @@ WatcherCheckResult FileWatcherCheckFileContentModified(void *state) bool hashed = false; unsigned char digest[EVP_MAX_MD_SIZE + 1] = { 0 }; - if (exists && S_ISREG(sb.st_mode)) + const StatSignature signature = exists ? StatSignatureOf(&sb) : (StatSignature) { 0 }; + if (exists && S_ISREG(sb.st_mode) && fws->last.content.hashed && + memcmp(&fws->last.content.signature, &signature, sizeof(signature)) == 0) + { + /* Unchanged since hashed, see StatSignatureOf() */ + hashed = true; + memcpy(digest, fws->last.content.digest, sizeof(digest)); + } + else if (exists && S_ISREG(sb.st_mode)) { if (HashFileContent(fws->path, digest)) { @@ -354,6 +509,7 @@ WatcherCheckResult FileWatcherCheckFileContentModified(void *state) fws->last.content.hashed = hashed; memcpy(fws->last.content.digest, digest, sizeof(digest)); + fws->last.content.signature = signature; return FileWatcherRecordObservation(fws, exists, modified, "content modification"); } diff --git a/cf-reactor/reactor_transform.c b/cf-reactor/reactor_transform.c index 451a68adf5c..af3fa817c44 100644 --- a/cf-reactor/reactor_transform.c +++ b/cf-reactor/reactor_transform.c @@ -150,6 +150,7 @@ typedef struct WatcherStateNewFn new_state; WatcherStateDestroyFn destroy_state; WatcherCheckFn check; + ReactorWatchTargetType target; /* what the 'name' constraint is about */ } EventType; static const EventArg NO_EXTRA_ARGS[] = @@ -163,17 +164,20 @@ static const EventArg FILE_SIZE_EXCEEDS_ARGS[] = { NULL, RVAL_TYPE_NOPROMISEE, false } }; -#define EventTypeNew(name, type, args, new_state, destroy_state, check) \ - { name, type, args, new_state, destroy_state, check } -#define EventTypeNewNull() EventTypeNew(NULL, RVAL_TYPE_NOPROMISEE, NULL, NULL, NULL, NULL) +#define EventTypeNew(name, type, args, new_state, destroy_state, check, target) \ + { name, type, args, new_state, destroy_state, check, target } +#define EventTypeNewNull() \ + EventTypeNew(NULL, RVAL_TYPE_NOPROMISEE, NULL, NULL, NULL, NULL, REACTOR_WATCH_TARGET_FILE) /* A file event, about the file at the path of its constraint */ #define FileEventTypeNew(name, check) \ - EventTypeNew(name, RVAL_TYPE_SCALAR, NO_EXTRA_ARGS, FileWatcherStateNew, FileWatcherStateDestroy, check) + EventTypeNew(name, RVAL_TYPE_SCALAR, NO_EXTRA_ARGS, FileWatcherStateNew, FileWatcherStateDestroy, \ + check, REACTOR_WATCH_TARGET_FILE) /* A user event, about the user with the name of its constraint */ #define UserEventTypeNew(name, check) \ - EventTypeNew(name, RVAL_TYPE_SCALAR, NO_EXTRA_ARGS, UserWatcherStateNew, UserWatcherStateDestroy, check) + EventTypeNew(name, RVAL_TYPE_SCALAR, NO_EXTRA_ARGS, UserWatcherStateNew, UserWatcherStateDestroy, \ + check, REACTOR_WATCH_TARGET_USERS) static const EventType EVENT_TABLE[] = { @@ -184,7 +188,8 @@ static const EventType EVENT_TABLE[] = FileEventTypeNew(REACTOR_FILE_PERMISSIONS_CHANGED, FileWatcherCheckFilePermissionsChanged), FileEventTypeNew(REACTOR_FILE_OWNER_CHANGED, FileWatcherCheckFileOwnerChanged), EventTypeNew(REACTOR_FILE_SIZE_EXCEEDS, RVAL_TYPE_SCALAR, FILE_SIZE_EXCEEDS_ARGS, - FileWatcherSizeStateNew, FileWatcherStateDestroy, FileWatcherCheckFileSizeExceeds), + FileWatcherSizeStateNew, FileWatcherStateDestroy, FileWatcherCheckFileSizeExceeds, + REACTOR_WATCH_TARGET_FILE), UserEventTypeNew(REACTOR_USER_ADDED, UserWatcherCheckUserAdded), UserEventTypeNew(REACTOR_USER_REMOVED, UserWatcherCheckUserRemoved), EventTypeNewNull() @@ -273,6 +278,31 @@ static bool EventsPromiseGetInterval(const Promise *pp, time_t *interval) return true; } +/** Sets *methods to the 'wakeup_sources' of the (expanded) events promise, NULL + * if it has none. The method names were checked against the syntax already, + * on expansion. */ +static bool EventsPromiseGetWakeupSources(const Promise *pp, const Rlist **methods) +{ + assert(pp != NULL); + assert(methods != NULL); + + *methods = NULL; + const Constraint *con = PromiseGetConstraint(pp, REACTOR_WAKEUP_SOURCES); + if (con == NULL) + { + return true; + } + if (con->rval.type != RVAL_TYPE_LIST) + { + Log(LOG_LEVEL_ERR, + "Reactor events promise '%s' has an invalid '%s', expected a list of wakeup sources", + pp->promiser, REACTOR_WAKEUP_SOURCES); + return false; + } + *methods = RvalRlistValue(con->rval); + return true; +} + static bool ResolveWhenBody(const EvalContext *ctx, const Promise *pp, WatcherOptions *opt) { assert(ctx != NULL); @@ -331,6 +361,7 @@ static bool ResolveWhenBody(const EvalContext *ctx, const Promise *pp, WatcherOp if (BodySyntaxGetConstraintSyntax(when_syntax, cp->lval) == NULL || /* not a 'when' constraint */ StringEqual(cp->lval, event->name) || StringEqual(cp->lval, "interval") || + StringEqual(cp->lval, REACTOR_WAKEUP_SOURCES) || StringEqual(cp->lval, "meta")) /* also a promise attribute */ { continue; @@ -351,12 +382,19 @@ static bool ResolveWhenBody(const EvalContext *ctx, const Promise *pp, WatcherOp } } - if (!EventsPromiseGetInterval(pp, &opt->interval)) + if (!EventsPromiseGetInterval(pp, &opt->interval) || + !EventsPromiseGetWakeupSources(pp, &opt->methods)) { RlistDestroy(args); return false; } + /* The path is the value of the event's constraint, which lives as long + * as pp, unlike args */ + opt->target.type = event->target; + opt->target.path = (event->target == REACTOR_WATCH_TARGET_FILE) ? + RvalScalarValue(PromiseGetConstraint(pp, event->name)->rval) : NULL; + opt->state = event->new_state(ctx, args); RlistDestroy(args); if (opt->state == NULL) diff --git a/cf-reactor/stoppable_thread.c b/cf-reactor/stoppable_thread.c index e4198e1235b..f9eabc7c719 100644 --- a/cf-reactor/stoppable_thread.c +++ b/cf-reactor/stoppable_thread.c @@ -25,8 +25,21 @@ #include #include #include -#include /* ThreadLock(), ThreadUnlock(), ThreadWait() */ +#include /* ThreadLock(), ThreadUnlock(), THREAD_BLOCK_INDEFINITELY */ #include +#include /* clock_gettime() */ + +/* The clock the timed waits on cond are measured against. The monotonic clock + * isn't affected by steps of the wall clock (NTP, an administrator, a resumed + * VM), so stepping it back doesn't stretch a sleep by the size of the step. + * mingw is excluded, as winpthreads accepts the attribute but may not honor + * it, which would make every timed wait return immediately. */ +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(CLOCK_MONOTONIC) && !defined(__MINGW32__) +# define STOPPABLE_THREAD_CLOCK CLOCK_MONOTONIC +# define STOPPABLE_THREAD_CLOCK_IS_MONOTONIC 1 +#else +# define STOPPABLE_THREAD_CLOCK CLOCK_REALTIME +#endif struct StoppableThread_ { @@ -73,7 +86,14 @@ StoppableThread *StoppableThreadStart(StoppableThreadFn fn, void *arg) thread->fn = fn; thread->arg = arg; pthread_mutex_init(&thread->lock, NULL); - pthread_cond_init(&thread->cond, NULL); + + pthread_condattr_t cond_attr; + pthread_condattr_init(&cond_attr); +#ifdef STOPPABLE_THREAD_CLOCK_IS_MONOTONIC + pthread_condattr_setclock(&cond_attr, STOPPABLE_THREAD_CLOCK); +#endif + pthread_cond_init(&thread->cond, &cond_attr); + pthread_condattr_destroy(&cond_attr); int ret = pthread_create(&thread->thread, NULL, StoppableThreadMain, thread); if (ret != 0) @@ -96,6 +116,46 @@ bool StoppableThreadShouldStop(StoppableThread *thread) return ret; } +/** + * Seconds on a clock that only moves forward, for scheduling and deadlines, so + * that stepping the wall clock back doesn't postpone everything. Also the + * clock the cond of a StoppableThread waits on. Falls back to the wall clock + * where there is no monotonic clock, or if reading it fails. + */ +time_t MonotonicClockNow(void) +{ + struct timespec ts = { 0 }; + if (clock_gettime(STOPPABLE_THREAD_CLOCK, &ts) != 0) + { + return time(NULL); + } + return ts.tv_sec; +} + +/** + * Like ThreadWait() on thread->cond, but with the deadline on the clock cond + * was created with (ThreadWait() uses the wall clock). Needs thread->lock held. + * + * @param timeout seconds, or THREAD_BLOCK_INDEFINITELY + */ +static void StoppableThreadCondWait(StoppableThread *thread, int timeout) +{ + assert(thread != NULL); + + if (timeout == THREAD_BLOCK_INDEFINITELY) + { + pthread_cond_wait(&thread->cond, &thread->lock); + return; + } + + struct timespec deadline; + clock_gettime(STOPPABLE_THREAD_CLOCK, &deadline); + deadline.tv_sec += timeout; + /* Timing out is the normal end of a sleep, and spurious wakeups are + * handled by the callers re-checking their state */ + pthread_cond_timedwait(&thread->cond, &thread->lock, &deadline); +} + /* timeout is in seconds, or THREAD_BLOCK_INDEFINITELY */ static void StoppableThreadWaitForWakeup(StoppableThread *thread, int timeout) { @@ -104,7 +164,7 @@ static void StoppableThreadWaitForWakeup(StoppableThread *thread, int timeout) ThreadLock(&thread->lock); if (!thread->stop_requested && !thread->wake_requested) { - ThreadWait(&thread->cond, &thread->lock, timeout); + StoppableThreadCondWait(thread, timeout); } /* The routine re-checks its state after any sleep, which covers the * wakeup, whether it arrived before or during the sleep */ @@ -146,11 +206,11 @@ bool StoppableThreadStop(StoppableThread *thread, int timeout_secs) thread->stop_requested = true; pthread_cond_broadcast(&thread->cond); - const time_t deadline = time(NULL) + timeout_secs; + const time_t deadline = MonotonicClockNow() + timeout_secs; time_t now; - while (!thread->exited && (now = time(NULL)) < deadline) + while (!thread->exited && (now = MonotonicClockNow()) < deadline) { - ThreadWait(&thread->cond, &thread->lock, (int) (deadline - now)); + StoppableThreadCondWait(thread, (int) (deadline - now)); } const bool exited = thread->exited; ThreadUnlock(&thread->lock); diff --git a/cf-reactor/stoppable_thread.h b/cf-reactor/stoppable_thread.h index fd08d0ff485..a3b1a819fdc 100644 --- a/cf-reactor/stoppable_thread.h +++ b/cf-reactor/stoppable_thread.h @@ -37,6 +37,9 @@ */ typedef struct StoppableThread_ StoppableThread; +/** Seconds on a monotonic clock (wall clock where there is none) */ +time_t MonotonicClockNow(void); + typedef void (*StoppableThreadFn)(StoppableThread *thread, void *arg); /** diff --git a/cf-reactor/watch_backends.c b/cf-reactor/watch_backends.c new file mode 100644 index 00000000000..d15d603a80e --- /dev/null +++ b/cf-reactor/watch_backends.c @@ -0,0 +1,262 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#include +#include /* MonotonicClockNow() */ +#include /* REACTOR_WAKEUP_SOURCE_POLL */ +#include /* ReactorEnterpriseWatchBackends() */ +#include +#include +#include /* StringEqual() */ +#include /* SeqStringFromString() */ + +#define MAX_WATCH_BACKENDS 16 + +/* A method that failed to start (e.g. out of fanotify marks or locked memory) + * is tried again when a watcher asks for it after this long */ +#define BACKEND_START_RETRY_SECS 60 + +typedef enum +{ + BACKEND_NOT_STARTED, + BACKEND_STARTED, + BACKEND_UNAVAILABLE, /* not supported here */ + BACKEND_FAILED, /* start() failed at `failed_at` */ +} BackendStatus; + +typedef struct +{ + const ReactorWatchBackend *backend; + BackendStatus status; + time_t failed_at; /* on MonotonicClockNow()'s clock */ +} BackendEntry; + +static BackendEntry backends[MAX_WATCH_BACKENDS]; +static size_t num_backends = 0; +static const ReactorWatchHost *watch_host = NULL; + +static void BackendAppend(const ReactorWatchBackend *backend) +{ + assert(backend != NULL); + assert(backend->name != NULL); + + if (num_backends >= MAX_WATCH_BACKENDS) + { + Log(LOG_LEVEL_ERR, "Too many cf-reactor wakeup sources, ignoring '%s'", backend->name); + return; + } + backends[num_backends].backend = backend; + backends[num_backends].status = BACKEND_NOT_STARTED; + backends[num_backends].failed_at = 0; + num_backends++; +} + +void WatchBackendsInitialize(const ReactorWatchHost *host) +{ + assert(host != NULL); + assert(num_backends == 0); + + watch_host = host; + + const ReactorWatchBackend *const *extra = ReactorEnterpriseWatchBackends(); + for (size_t i = 0; extra != NULL && extra[i] != NULL; i++) + { + BackendAppend(extra[i]); + } + + for (size_t i = 0; i < num_backends; i++) + { + Log(LOG_LEVEL_DEBUG, "cf-reactor wakeup source '%s' is built in", backends[i].backend->name); + } +} + +void WatchBackendsFinalize(void) +{ + for (size_t i = 0; i < num_backends; i++) + { + if (backends[i].status == BACKEND_STARTED) + { + backends[i].backend->stop(); + } + } + num_backends = 0; + watch_host = NULL; +} + +static BackendEntry *BackendLookup(const char *name) +{ + assert(name != NULL); + + for (size_t i = 0; i < num_backends; i++) + { + if (StringEqual(backends[i].backend->name, name)) + { + return &backends[i]; + } + } + return NULL; +} + +/** + * Starts the method on first use, so that methods no policy asks for (e.g. + * eBPF, which loads a program into the kernel) are never set up + */ +static bool BackendEnsureStarted(BackendEntry *entry) +{ + assert(entry != NULL); + assert(watch_host != NULL); + + if (entry->status == BACKEND_FAILED && + MonotonicClockNow() - entry->failed_at >= BACKEND_START_RETRY_SECS) + { + entry->status = BACKEND_NOT_STARTED; + } + + if (entry->status == BACKEND_NOT_STARTED) + { + const ReactorWatchBackend *b = entry->backend; + switch (b->start(watch_host)) + { + case REACTOR_WATCH_STARTED: + Log(LOG_LEVEL_VERBOSE, "Started cf-reactor wakeup source '%s'", b->name); + entry->status = BACKEND_STARTED; + break; + case REACTOR_WATCH_UNSUPPORTED: + Log(LOG_LEVEL_VERBOSE, "cf-reactor wakeup source '%s' is not available on this host", b->name); + entry->status = BACKEND_UNAVAILABLE; + break; + case REACTOR_WATCH_START_FAILED: + Log(LOG_LEVEL_ERR, "Failed to start cf-reactor wakeup source '%s', trying it again in %d seconds", + b->name, BACKEND_START_RETRY_SECS); + entry->status = BACKEND_FAILED; + entry->failed_at = MonotonicClockNow(); + break; + } + } + return (entry->status == BACKEND_STARTED); +} + +/** + * Tries method `name` for the watcher, returns its static name on success. + * `requested` is whether the policy named the method, rather than it coming + * from the default list: then a method missing from this build is worth + * telling at verbose level, it's why the watcher may end up with none. + */ +static const char *TryMethod(const char *name, uint64_t watcher_id, + const ReactorWatchTarget *target, const char *promiser, + bool requested) +{ + assert(name != NULL); + assert(target != NULL); + + if (StringEqual(name, REACTOR_WAKEUP_SOURCE_POLL)) + { + return REACTOR_WAKEUP_SOURCE_POLL; + } + + BackendEntry *entry = BackendLookup(name); + if (entry == NULL) + { + Log(requested ? LOG_LEVEL_VERBOSE : LOG_LEVEL_DEBUG, + "Reactor watcher '%s': wakeup source '%s' isn't part of this build, trying the next one", + promiser, name); + return NULL; + } + if (!BackendEnsureStarted(entry)) + { + return NULL; + } + if (!entry->backend->add(watcher_id, target)) + { + Log(LOG_LEVEL_VERBOSE, + "Reactor watcher '%s': wakeup source '%s' can't watch this, trying the next one", + promiser, name); + return NULL; + } + return entry->backend->name; +} + +const char *WatchBackendsAdd(uint64_t watcher_id, const ReactorWatchTarget *target, + const Rlist *methods, const char *promiser) +{ + assert(target != NULL); + assert(promiser != NULL); + + if (methods == NULL) + { + const char *override = getenv(WAKEUP_SOURCES_TEST_OVERRIDE_ENV); + if (override != NULL && override[0] != '\0') + { + Seq *names = SeqStringFromString(override, ','); + const char *used = NULL; + bool poll_allowed = false; + const size_t length = SeqLength(names); + for (size_t i = 0; i < length && used == NULL; i++) + { + const char *name = SeqAt(names, i); + poll_allowed = poll_allowed || StringEqual(name, REACTOR_WAKEUP_SOURCE_POLL); + used = TryMethod(name, watcher_id, target, promiser, true); + } + SeqDestroy(names); + return (used != NULL || !poll_allowed) ? used : REACTOR_WAKEUP_SOURCE_POLL; + } + + /* By default every method in the build, in the order they're built in + * (closest to the kernel first), then polling */ + for (size_t i = 0; i < num_backends; i++) + { + const char *used = TryMethod(backends[i].backend->name, watcher_id, target, promiser, + false); + if (used != NULL) + { + return used; + } + } + return REACTOR_WAKEUP_SOURCE_POLL; + } + + for (const Rlist *rp = methods; rp != NULL; rp = rp->next) + { + const char *used = TryMethod(RlistScalarValue(rp), watcher_id, target, promiser, true); + if (used != NULL) + { + return used; + } + } + return NULL; +} + +void WatchBackendsRemove(uint64_t watcher_id, const char *method) +{ + if (method == NULL || StringEqual(method, REACTOR_WAKEUP_SOURCE_POLL)) + { + return; + } + + BackendEntry *entry = BackendLookup(method); + if (entry != NULL && entry->status == BACKEND_STARTED) + { + entry->backend->remove(watcher_id); + } +} diff --git a/cf-reactor/watch_backends.h b/cf-reactor/watch_backends.h new file mode 100644 index 00000000000..1e14552dd3d --- /dev/null +++ b/cf-reactor/watch_backends.h @@ -0,0 +1,64 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef CFENGINE_WATCH_BACKENDS_H +#define CFENGINE_WATCH_BACKENDS_H + +#include +#include + +/* Environment variable replacing the default 'wakeup_sources' (of watchers + * that don't set it), a comma-separated list, so that the reactor tests can + * be run with each method in turn */ +#define WAKEUP_SOURCES_TEST_OVERRIDE_ENV "CFENGINE_TEST_OVERRIDE_WAKEUP_SOURCES" + +/** + * @brief Collect the wakeup sources the Enterprise extension provides, if + * any. They are started when first used. + * @param host what the methods call back into, see ReactorWatchHost + */ +void WatchBackendsInitialize(const ReactorWatchHost *host); + +/** @brief Stop the started methods. */ +void WatchBackendsFinalize(void); + +/** + * @brief Watch target with the first available method of methods. + * + * Methods for another platform, or that this binary doesn't have, are + * skipped. "poll" always succeeds, as the watcher is polled at its interval + * anyway: it means polling alone is acceptable. + * + * @param watcher_id the id passed back to ReactorWatchHost.mark_dirty() + * @param methods method names in order of preference, NULL for the default + * @return the name of the method used (static), or NULL if none of methods + * is available and they don't include "poll" + */ +const char *WatchBackendsAdd(uint64_t watcher_id, const ReactorWatchTarget *target, + const Rlist *methods, const char *promiser); + +/** @brief Stop watching for watcher_id with method, as returned by WatchBackendsAdd(). */ +void WatchBackendsRemove(uint64_t watcher_id, const char *method); + +#endif diff --git a/cf-reactor/watch_location.c b/cf-reactor/watch_location.c new file mode 100644 index 00000000000..8590715105b --- /dev/null +++ b/cf-reactor/watch_location.c @@ -0,0 +1,466 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +/* Whether a watched file is on a remote file system, whose checks may block + * for as long as the server doesn't respond, so they get their own thread + * (see watcher.c). Only ever decided from the path and the mount table: any + * call on the path itself (stat(), realpath()) could block just the same. */ + +#include +#include +#include +#include /* StringEqual(), StringStartsWith(), IsStrIn() */ +#include /* FileReadFromFd() */ +#include /* StringWriterClose() */ + +/* More than any real mount table, against reading forever */ +#define MOUNTINFO_MAX_SIZE (4 * 1024 * 1024) + +/* Field 5 of a line of /proc/self/mountinfo (counting from 1) is the mount + * point, and the file system type follows the "-" ending the optional + * fields. See proc(5). */ +#define MOUNTINFO_MOUNT_POINT_FIELD 5 + +/** + * Whether files of this type may hang when their server doesn't respond. + * FUSE counts as remote: its daemon can hang just the same, whatever it + * serves, and so can virtiofs's on the host. + */ +static bool FsTypeIsRemote(const char *fstype) +{ + assert(fstype != NULL); + + static const char *const REMOTE_FS_TYPES[] = { + "nfs", "nfs4", "cifs", "smb3", "smbfs", "ncpfs", "9p", "ceph", "glusterfs", + "afs", "lustre", "beegfs", "panfs", "ocfs2", "gfs2", "virtiofs", + "fuse", "fuseblk", NULL + }; + return IsStrIn(fstype, REMOTE_FS_TYPES) || StringStartsWith(fstype, "fuse."); +} + +/** + * Decodes, in place, the octal escapes mountinfo writes for the characters + * that would break its format, e.g. "\040" for a space. + */ +static void MountinfoUnescape(char *s) +{ + assert(s != NULL); + + char *out = s; + const char *in = s; + while (*in != '\0') + { + if (in[0] == '\\' && + in[1] >= '0' && in[1] <= '3' && + in[2] >= '0' && in[2] <= '7' && + in[3] >= '0' && in[3] <= '7') + { + *out++ = (char) (((in[1] - '0') << 6) | ((in[2] - '0') << 3) | (in[3] - '0')); + in += 4; + } + else + { + *out++ = *in++; + } + } + *out = '\0'; +} + +typedef struct +{ + char *mount_point; + size_t length; + bool remote; +} MountEntry; + +/* The mount points of a mount table, in mount order */ +typedef struct +{ + MountEntry *entries; + size_t count; +} MountTable; + +/** + * Finds, in place, the mount point and the file system type of a line of + * mountinfo, the only fields used. Returns false for a line it can't make + * sense of. + */ +static bool MountinfoParseLine(char *line, char **mount_point, const char **fstype) +{ + assert(line != NULL); + assert(mount_point != NULL); + assert(fstype != NULL); + + char *field = line; + for (int i = 1; i < MOUNTINFO_MOUNT_POINT_FIELD; i++) + { + field = strchr(field, ' '); + if (field == NULL) + { + return false; + } + field++; + } + char *end = strchr(field, ' '); + if (end == NULL) + { + return false; + } + *end = '\0'; + *mount_point = field; + + /* Past the mount options and the optional fields, to the "-" */ + for (char *next = end + 1; ; ) + { + char *separator = strchr(next, ' '); + if (separator == NULL) + { + return false; + } + if (separator == next + 1 && next[0] == '-') + { + char *type = separator + 1; + char *type_end = strchr(type, ' '); + if (type_end != NULL) + { + *type_end = '\0'; + } + *fstype = type; + return (type[0] != '\0'); + } + next = separator + 1; + } +} + +static MountTable *MountTableParse(const char *mountinfo) +{ + assert(mountinfo != NULL); + + MountTable *table = xcalloc(1, sizeof(MountTable)); + size_t capacity = 0; + char *text = xstrdup(mountinfo); + for (char *line = text; line != NULL && *line != '\0'; ) + { + char *line_end = strchr(line, '\n'); + if (line_end != NULL) + { + *line_end = '\0'; + } + + char *mount_point; + const char *fstype; + if (MountinfoParseLine(line, &mount_point, &fstype)) + { + if (table->count == capacity) + { + capacity = (capacity == 0) ? 64 : capacity * 2; + table->entries = xrealloc(table->entries, capacity * sizeof(MountEntry)); + } + MountinfoUnescape(mount_point); + MountEntry *entry = &table->entries[table->count++]; + entry->mount_point = xstrdup(mount_point); + entry->length = strlen(mount_point); + entry->remote = FsTypeIsRemote(fstype); + } + line = (line_end != NULL) ? line_end + 1 : NULL; + } + free(text); + return table; +} + +static void MountTableDestroy(MountTable *table) +{ + if (table != NULL) + { + for (size_t i = 0; i < table->count; i++) + { + free(table->entries[i].mount_point); + } + free(table->entries); + free(table); + } +} + +/** + * Length of the mount point of `entry` if `path` is under it (or is it), + * else 0. Only compares whole components, so /mnt/nfs doesn't contain + * /mnt/nfs2. + */ +static size_t MountPointMatch(const MountEntry *entry, const char *path) +{ + assert(entry != NULL); + assert(path != NULL); + + if (StringEqual(entry->mount_point, "/")) + { + return 1; + } + const size_t length = entry->length; + if (length == 0 || !StringEqualN(entry->mount_point, path, length)) + { + return 0; + } + return (path[length] == '/' || path[length] == '\0') ? length : 0; +} + +/** + * The absolute `path` without repeated or trailing slashes, "." or "..", as + * in the mount points. Lexical: "/mnt/link/.." is "/mnt" even if link is a + * symlink elsewhere, where the kernel would resolve the link first. + */ +static char *NormalizePath(const char *path) +{ + assert(path != NULL); + assert(path[0] == '/'); + + char *normalized = xmalloc(strlen(path) + 2); + size_t length = 0; + const char *component = path; + while (*component != '\0') + { + if (*component == '/') + { + component++; + continue; + } + const char *slash = strchr(component, '/'); + const size_t component_length = (slash != NULL) ? (size_t) (slash - component) : + strlen(component); + if (component_length == 2 && component[0] == '.' && component[1] == '.') + { + /* Drop the last component, the root's parent is the root */ + while (length > 0 && normalized[length - 1] != '/') + { + length--; + } + if (length > 0) + { + length--; + } + } + else if (component_length != 1 || component[0] != '.') + { + normalized[length++] = '/'; + memcpy(normalized + length, component, component_length); + length += component_length; + } + component += component_length; + } + if (length == 0) + { + normalized[length++] = '/'; + } + normalized[length] = '\0'; + return normalized; +} + +/** + * Whether `path` is on a remote file system according to `table`: that of the + * longest mount point containing it, the last mounted of equal ones, as it + * hides those below. Purely lexical, so a symlink to a remote file system + * isn't seen, nor is a relative path. + */ +static bool MountTablePathIsRemote(const MountTable *table, const char *path) +{ + assert(table != NULL); + assert(path != NULL); + + if (path[0] != '/') + { + return false; + } + + char *normalized = NormalizePath(path); + size_t longest = 0; + bool remote = false; + for (size_t i = 0; i < table->count; i++) + { + const size_t match = MountPointMatch(&table->entries[i], normalized); + if (match > 0 && match >= longest) + { + longest = match; + remote = table->entries[i].remote; + } + } + free(normalized); + return remote; +} + +/** + * Whether `path` is on a remote file system according to `mountinfo`, the + * text of /proc/self/mountinfo, see MountTablePathIsRemote() + */ +bool MountinfoPathIsRemote(const char *mountinfo, const char *path) +{ + assert(mountinfo != NULL); + assert(path != NULL); + + MountTable *table = MountTableParse(mountinfo); + const bool remote = MountTablePathIsRemote(table, path); + MountTableDestroy(table); + return remote; +} + +static bool IsPathSeparator(char c) +{ + return (c == '\\' || c == '/'); +} + +/** + * Whether `path` is a Windows UNC path (\\server\share, //server/share, + * \\?\UNC\server\share). Lexical, so available everywhere for the tests. + */ +bool WatchPathIsUNC(const char *path) +{ + assert(path != NULL); + + if (!IsPathSeparator(path[0]) || !IsPathSeparator(path[1])) + { + return false; + } + if ((path[2] == '?' || path[2] == '.') && IsPathSeparator(path[3])) + { + /* A long path (\\?\C:\...) or a device (\\.\...), unless \\?\UNC\ */ + return (path[2] == '?' && strncasecmp(path + 4, "UNC", 3) == 0 && + IsPathSeparator(path[7])); + } + return (path[2] != '\0'); +} + +#ifdef __MINGW32__ +static bool WindowsPathIsRemote(const char *path) +{ + assert(path != NULL); + + if (WatchPathIsUNC(path)) + { + return true; + } + + const char *drive = path; + if (IsPathSeparator(path[0]) && IsPathSeparator(path[1]) && path[2] == '?' && + IsPathSeparator(path[3])) + { + drive = path + 4; + } + if (!isalpha((unsigned char) drive[0]) || drive[1] != ':') + { + return false; + } + /* Asks about the drive, not the path, so it doesn't wait for its server */ + const wchar_t root[] = { (wchar_t) drive[0], L':', L'\\', L'\0' }; + return (GetDriveTypeW(root) == DRIVE_REMOTE); +} +#else /* !__MINGW32__ */ +/** + * The text of the mount table in the format of /proc/self/mountinfo, NULL if + * there is none (no other Unix has one we can read without risking to block). + * A procfs file, so reading it doesn't block. + */ +static char *ReadMountTable(void) +{ +# ifdef __linux__ + const char *const file = "/proc/self/mountinfo"; + const int fd = open(file, O_RDONLY); + if (fd == -1) + { + Log(LOG_LEVEL_VERBOSE, "Failed to open the mount table '%s' (open: %s)", file, GetErrorStr()); + return NULL; + } + bool truncated = false; + Writer *contents = FileReadFromFd(fd, MOUNTINFO_MAX_SIZE, &truncated); + close(fd); + if (contents == NULL) + { + Log(LOG_LEVEL_VERBOSE, "Failed to read the mount table '%s'", file); + return NULL; + } + if (truncated) + { + Log(LOG_LEVEL_VERBOSE, "The mount table '%s' is larger than %d bytes, only reading the start", + file, MOUNTINFO_MAX_SIZE); + } + return StringWriterClose(contents); +# else + return NULL; +# endif +} +#endif /* !__MINGW32__ */ + +/* The mount table, read and parsed once per policy (re-)read, as all its + * watchers are registered at once, NULL until needed. Only used by the main + * thread. */ +static MountTable *mount_table = NULL; + +/* Replaces the mount table of the host, for the tests */ +static char *mount_table_override = NULL; + +/** + * Forgets the mount table, so that it is read again for the next watcher. + * Called once the watchers of a policy are registered. + */ +void WatchLocationForgetMountTable(void) +{ + MountTableDestroy(mount_table); + mount_table = NULL; +} + +/** + * Makes WatchTargetIsRemote() use `mountinfo`, in the format of + * /proc/self/mountinfo, instead of the mount table of the host, until called + * with NULL. For the tests, not used on Windows (no mount table). + */ +void WatchLocationSetMountTable(const char *mountinfo) +{ + free(mount_table_override); + mount_table_override = (mountinfo != NULL) ? xstrdup(mountinfo) : NULL; + WatchLocationForgetMountTable(); +} + +/** + * Whether checks of the watcher of `target` may block on a remote file system. + * Only files can be remote. Without a mount table, everything is local. + */ +bool WatchTargetIsRemote(const ReactorWatchTarget *target) +{ + assert(target != NULL); + + if (target == NULL || target->type != REACTOR_WATCH_TARGET_FILE || target->path == NULL) + { + return false; + } + +#ifdef __MINGW32__ + return WindowsPathIsRemote(target->path); +#else + if (mount_table == NULL) + { + char *text = (mount_table_override == NULL) ? ReadMountTable() : NULL; + const char *mountinfo = (mount_table_override != NULL) ? mount_table_override : text; + mount_table = MountTableParse((mountinfo != NULL) ? mountinfo : ""); + free(text); + } + return MountTablePathIsRemote(mount_table, target->path); +#endif +} diff --git a/cf-reactor/watch_location.h b/cf-reactor/watch_location.h new file mode 100644 index 00000000000..d296701b106 --- /dev/null +++ b/cf-reactor/watch_location.h @@ -0,0 +1,39 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef CFENGINE_WATCH_LOCATION_H +#define CFENGINE_WATCH_LOCATION_H + +#include +#include /* ReactorWatchTarget */ + +bool WatchTargetIsRemote(const ReactorWatchTarget *target); +void WatchLocationForgetMountTable(void); + +/** For the tests */ +void WatchLocationSetMountTable(const char *mountinfo); +bool MountinfoPathIsRemote(const char *mountinfo, const char *path); +bool WatchPathIsUNC(const char *path); + +#endif diff --git a/cf-reactor/watcher.c b/cf-reactor/watcher.c index 1073f68066c..56e27e0129e 100644 --- a/cf-reactor/watcher.c +++ b/cf-reactor/watcher.c @@ -37,40 +37,197 @@ #include // PromiseRuntimeHash() #include // HashPrintSafe() #include // Promise - -/* Upper bound on how long the watcher thread sleeps in one go while it has - * watchers to check, regardless of what poll intervals they asked for. When - * it has none it sleeps until woken up. Either way, shutdown doesn't depend - * on it: EventWatcherFinalize() wakes the thread up to stop it. */ -#define MAX_WATCHER_THREAD_SLEEP_SECS 1 +#include // REACTOR_WAKEUP_SOURCES +#include // WatchBackendsAdd(), WatchBackendsRemove() +#include +#include // SeqSort() +#include // WatchersReportWrite() +#include // WatchTargetIsRemote(), WatchLocationForgetMountTable() + +/* Longest a lane sleeps before its watchers are due, as a safety net against + * a lost wakeup or a clock step. With no watchers it sleeps until woken up. + * The intervals are on MonotonicClockNow()'s clock, CLOCK_MONOTONIC where + * available, which on Linux stops while the system is suspended: after a + * resume, a watcher is due up to its interval later than by the wall clock + * (the wakeup sources still wake it up as soon as something changes). */ +#define MAX_WATCHER_THREAD_SLEEP_SECS 30 #define WATCHER_THREAD_EXIT_TIMEOUT_SECS 5 +/* Wait after a wakeup source marked watchers dirty, before checking them. One + * operation is often several changes (a file created, then its times set), and + * checking in between would see states polling never sees. */ +#define WATCH_SETTLE_MSECS 200 + +/* A watcher whose check failed (e.g. its file locked on Windows) is retried + * after this long, not after its interval: until a check succeeds it has no + * reference, so a change before that would become the reference, not fire. */ +#define CHECK_ERROR_RETRY_SECS 1 + +/* Limit on how often a watcher runs its 'then' bundles: wakeup sources wake a + * watcher on every change, so anyone able to touch a watched file could make + * the bundles run continuously. Beyond MAX_TRIGGERS_PER_WINDOW runs in + * TRIGGER_WINDOW_SECS the watcher waits before its next run, twice as long + * each time (up to MAX_TRIGGER_BACKOFF_SECS), until a window stays below the + * limit. Events meanwhile are coalesced into the next run. */ +#define TRIGGER_WINDOW_SECS 60 +#define MAX_TRIGGERS_PER_WINDOW 10 +#define MIN_TRIGGER_BACKOFF_SECS 10 +#define MAX_TRIGGER_BACKOFF_SECS 300 + +/* State of the limit on how often a watcher runs its 'then' bundles, see + * ThrottleMayRun() */ +typedef struct +{ + time_t window_start; + unsigned int triggers_in_window; + time_t backoff_secs; + time_t throttled_until; // 0 when not throttled +} Throttle; + +static void ThrottleInit(Throttle *t, time_t now) +{ + assert(t != NULL); + + memset(t, 0, sizeof(*t)); + t->window_start = now; +} + +/** Seconds left of the delay after the limit was hit, 0 if not delayed */ +static time_t ThrottleDelayLeft(const Throttle *t, time_t now) +{ + assert(t != NULL); + + return (t->throttled_until > now) ? (t->throttled_until - now) : 0; +} + +/** Whether the watcher may run its bundles now, see TRIGGER_WINDOW_SECS. + * Counts the run if so. `promiser` is for the log. */ +static bool ThrottleMayRun(Throttle *t, const char *promiser, time_t now) +{ + assert(t != NULL); + assert(promiser != NULL); + + if (now - t->window_start >= TRIGGER_WINDOW_SECS) + { + if (t->triggers_in_window < MAX_TRIGGERS_PER_WINDOW) + { + /* A calm window ends the backoff */ + t->backoff_secs = 0; + } + t->window_start = now; + t->triggers_in_window = 0; + } + + if (t->throttled_until > now) + { + return false; + } + if (t->throttled_until != 0) + { + /* The delay is over: run once for the events held meanwhile */ + t->throttled_until = 0; + t->triggers_in_window++; + return true; + } + + if (t->triggers_in_window >= MAX_TRIGGERS_PER_WINDOW) + { + t->backoff_secs = (t->backoff_secs == 0) ? + MIN_TRIGGER_BACKOFF_SECS : MIN(t->backoff_secs * 2, MAX_TRIGGER_BACKOFF_SECS); + t->throttled_until = now + t->backoff_secs; + Log(LOG_LEVEL_WARNING, + "Reactor watcher '%s' fired more than %d times in %d seconds, delaying its next run by %jd seconds", + promiser, MAX_TRIGGERS_PER_WINDOW, TRIGGER_WINDOW_SECS, + (intmax_t) t->backoff_secs); + return false; + } + + t->triggers_in_window++; + return true; +} + +typedef struct WatcherLane_ WatcherLane; + +/* What the identical watchers (same promise hash) of successive policies + * share, so that re-reading the policy keeps what the watcher observed. See + * the invariants above watchers_mutex. */ typedef struct { + char *hash; // promise hash, see WatcherPromiseHash() + WatcherLane *lane; // the lane checking it, never changes WatcherCheckFn check_callback; WatcherStateDestroyFn destroy_state; void *state; - Promise *promise; // copy of the expanded events promise, see PromiseCopy() - char *promise_hash; // hash of `promise`, see WatcherPromiseHash() - time_t poll_interval_secs; - time_t next_due; - WatcherCheckResult last_check; // only touched by the watcher thread + time_t interval; // seconds between two checks + unsigned int refs; // watchers using it, plus one while its lane checks it + time_t next_due; // on MonotonicClockNow()'s clock + WatcherCheckResult last_check; + + /* An event was queued and its bundles haven't finished running. Events + * observed meanwhile are held, see EventHandled(). */ + bool event_pending; + + /* An observed event not queued yet: one is pending, the trigger limit + * delays it, or no registered watcher uses the slot (paused, or the + * re-read policy dropped the watcher while its check ran) */ + bool event_held; + + /* The wakeup source whose wakeup led to the check that observed the held + * event, NULL if it was the interval, for the log */ + const char *event_woken_by; + + /* To check as soon as it is of use, whether due or not: a check was + * skipped while an event was held, or its wakeups were dropped while + * paused. See WatcherWantsCheck(). */ + bool recheck; + Throttle throttle; +} StateSlot; + +typedef struct +{ + StateSlot *slot; // holds a reference to it + Promise *promise; // copy of the expanded events promise, see PromiseCopy() + uint64_t id; // passed to the wakeup sources, see LaneOfId() + const char *method; // wakeup source used, static, see WatchBackendsAdd() } Watcher; /* Item of the event queue */ typedef struct { - const Watcher *watcher; + Watcher *watcher; WatcherCheckResult check; // see WatcherEventFn + const char *woken_by; // copy of StateSlot.event_woken_by } WatcherEvent; static void WatcherDestroy(void *item); -/* Guards `watchers`, `old_watchers` and `paused` against the watcher thread - * (WatcherThreadMain()) iterating over `watchers` concurrently with the main - * thread re-registering watchers from a freshly (re-)read policy - * (WatcherRegister(), EventWatcherPause()). */ +/* Guards `watchers`, `old_watchers`, `paused` and the state slots, except + * the `state` of a pinned slot. The invariants: + * + * - Only the main thread creates and frees watchers (WatcherRegister(), + * EventWatcherPause(), EventWatcherResume(), finalizing), so only it calls + * the wakeup sources' add() and remove(). `watchers` maps the promise hash + * to the registered watcher, `old_watchers` holds those set aside. + * - A watcher holds a reference to a state slot. An identical watcher of the + * re-read policy takes a reference to the same slot, whatever the slot is + * doing, so the state, a held event and the trigger limit carry over. Not + * if it is on the other lane (the mount table changed, so the path is on + * another file system): it gets a fresh slot, as the old state describes + * the other file system, and it mustn't wait for a check stuck on it. So + * every watcher of a slot is on the slot's lane. + * - Only its lane checks a slot. It takes a reference to it under the lock + * (pins it), runs the check without the lock, as checks may block (an + * unresponsive NFS server, a big file to hash), then applies the result + * and drops the reference under the lock. While pinned, the `state` + * belongs to the check alone. + * - A slot is freed by whoever drops its last reference: the main thread + * freeing a watcher, or the lane after its check. Never while pinned. + * - Events are queued for the registered watcher of the slot + * (MapGet(watchers, hash)), and never while paused. Otherwise the slot + * holds the event, for a watcher registered later to queue. + * - Lock order: watchers_mutex, then the wakeup sources' locks, then + * dirty_mutex, then the lanes' StoppableThread locks. */ static pthread_mutex_t watchers_mutex = PTHREAD_MUTEX_INITIALIZER; static Map *watchers = NULL; static Map *old_watchers = NULL; @@ -78,28 +235,190 @@ static bool paused = false; /* see EventWatcherPause() */ static WakeupChannel wakeup_channel = { .fds = { -1, -1 } }; -/* Events of watchers (WatcherEvent), queued by the watcher thread for the - * main thread to handle. The watchers are not owned: the watcher thread only - * queues events of watchers of the registry while holding watchers_mutex, and - * EventWatcherPause() empties the queue before setting the watchers aside, so the watchers of - * the queued events are always registered. */ +/* Events (WatcherEvent) queued by the lanes for the main thread. The + * watchers aren't owned by the events: they are queued while holding + * watchers_mutex, and EventWatcherPause() empties the queue before setting + * the watchers aside, so the watchers of queued events are always registered. */ static ThreadedQueue *event_queue = NULL; -static StoppableThread *watcher_thread = NULL; + +/* Set of watcher ids: open addressing, linear probing, 0 is an empty cell + * (ids are never 0). Add and lookup are O(1), so a burst of marks is linear. */ +typedef struct +{ + uint64_t *cells; + size_t capacity; // 0 or a power of two + size_t count; +} IdSet; + +static size_t IdSetHome(uint64_t id, size_t capacity) +{ + return (size_t) ((id * UINT64_C(0x9E3779B97F4A7C15)) >> 32) & (capacity - 1); +} + +static bool IdSetContains(const IdSet *set, uint64_t id) +{ + assert(set != NULL); + + if (set->count == 0) + { + return false; + } + for (size_t i = IdSetHome(id, set->capacity); set->cells[i] != 0; + i = (i + 1) & (set->capacity - 1)) + { + if (set->cells[i] == id) + { + return true; + } + } + return false; +} + +static void IdSetAdd(IdSet *set, uint64_t id) +{ + assert(set != NULL); + assert(id != 0); + + if (IdSetContains(set, id)) + { + return; + } + if ((set->count + 1) * 2 > set->capacity) + { + const IdSet old = *set; + set->capacity = (old.capacity == 0) ? 16 : old.capacity * 2; + set->cells = xcalloc(set->capacity, sizeof(uint64_t)); + for (size_t i = 0; i < old.capacity; i++) + { + if (old.cells[i] != 0) + { + size_t j = IdSetHome(old.cells[i], set->capacity); + while (set->cells[j] != 0) + { + j = (j + 1) & (set->capacity - 1); + } + set->cells[j] = old.cells[i]; + } + } + free(old.cells); + } + size_t i = IdSetHome(id, set->capacity); + while (set->cells[i] != 0) + { + i = (i + 1) & (set->capacity - 1); + } + set->cells[i] = id; + set->count++; +} + +static void IdSetDestroy(IdSet *set) +{ + assert(set != NULL); + + free(set->cells); + *set = (IdSet) { 0 }; +} + +/** Keeps the capacity, to not reallocate on every burst */ +static void IdSetClear(IdSet *set) +{ + assert(set != NULL); + + if (set->count > 0) + { + memset(set->cells, 0, set->capacity * sizeof(uint64_t)); + set->count = 0; + } +} + +/* A slot pinned for a check in a pass of its lane */ +typedef struct +{ + StateSlot *slot; + WatcherCheckResult check; +} PinnedCheck; + +/* A thread checking its own watchers, see WatcherLaneMain() */ +struct WatcherLane_ +{ + const char *name; // for the log + StoppableThread *thread; // NULL while not running + + /* Left behind by EventWatcherFinalize(), stuck in a check, see LaneStop() */ + StoppableThread *stuck_thread; + + /* Ids of its watchers to check right away, marked by the wakeup sources, + * see WatcherMarkDirty() */ + IdSet dirty_ids; + bool all_dirty; + + /* Only used by its thread, not on its stack so they stay reachable if it + * is left behind (and cancelled), see LaneStop() */ + IdSet taken_ids; + PinnedCheck *pinned; + size_t pinned_capacity; +}; + +/* Guards the lanes' `thread`, `dirty_ids` and `all_dirty`. Only ever taken + * after watchers_mutex, never before: the wakeup sources mark watchers while + * holding their own lock, which is taken after watchers_mutex when + * (un)registering. The main thread alone starts and stops the lanes, so it + * may read `thread` without it. */ +static pthread_mutex_t dirty_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* The watchers whose checks may block on a remote file system have their + * own lane, started with the first of them, so that an unresponsive server + * doesn't hold the checks of the local ones up. */ +static WatcherLane local_lane = { .name = "local" }; +static WatcherLane remote_lane = { .name = "remote" }; + +/* Watcher ids are (n << 1) | 1 on the remote lane, (n << 1) on the local + * one, with n never reused (and never 0), so a mark is routed to its lane + * without a lookup */ +static uint64_t last_watcher_number = 0; +#define WATCHER_ID_REMOTE_BIT UINT64_C(1) + +static WatcherLane *LaneOfId(uint64_t id) +{ + return ((id & WATCHER_ID_REMOTE_BIT) != 0) ? &remote_lane : &local_lane; +} + +static const ReactorWatchHost WATCH_HOST = +{ + .mark_dirty = WatcherMarkDirty, + .mark_all_dirty = WatcherMarkAllDirty, +}; /*****************************************************************************/ /* The hash covers the whole expanded promise, including what is watched * (the constraints of the 'when' body) and what is run ('then'), so that a * state is only taken over by a watcher of the same type, on the same - * thing. */ + * thing. It leaves out 'wakeup_sources', which only sets how the event is + * watched for: changing it must not reset the state, or an event happening + * while the policy is re-read would be missed. */ void WatcherPromiseHash(const Promise *pp, char *hash, size_t hash_size) { assert(pp != NULL); assert(hash != NULL); + const size_t length = (pp->conlist == NULL) ? 0 : SeqLength(pp->conlist); + Seq *conlist = SeqNew(length, NULL); + for (size_t i = 0; i < length; i++) + { + Constraint *cp = SeqAt(pp->conlist, i); + if (!StringEqual(cp->lval, REACTOR_WAKEUP_SOURCES)) + { + SeqAppend(conlist, cp); + } + } + Promise hashed = *pp; + hashed.conlist = conlist; + unsigned char digest[EVP_MAX_MD_SIZE + 1]; - PromiseRuntimeHash(pp, "events_promise_hash", digest, CF_DEFAULT_DIGEST); + PromiseRuntimeHash(&hashed, "events_promise_hash", digest, CF_DEFAULT_DIGEST); HashPrintSafe(hash, hash_size, digest, CF_DEFAULT_DIGEST, true); + SeqDestroy(conlist); } void WatcherRegistryInitialize(void) @@ -107,6 +426,7 @@ void WatcherRegistryInitialize(void) assert(watchers == NULL); watchers = MapNew(StringHash_untyped, StringEqual_untyped, NULL, WatcherDestroy); + paused = false; /* if finalized while paused */ } void WatcherRegistryFinalize(void) @@ -117,7 +437,128 @@ void WatcherRegistryFinalize(void) old_watchers = NULL; } -// Expects interval to be strictly greater than 0, otherwise the watcher thread will busy spin +static void WatcherLaneMain(StoppableThread *thread, void *arg); + +/** Wakes the lane up to check its watchers, holding dirty_mutex */ +static void LaneWakeLocked(WatcherLane *lane) +{ + assert(lane != NULL); + + if (lane->thread != NULL) + { + StoppableThreadWake(lane->thread); + } +} + +/** Wakes the lane up to check its watchers. Thread-safe. */ +static void LaneWake(WatcherLane *lane) +{ + /* Under the lock, as EventWatcherFinalize() may be stopping the thread */ + ThreadLock(&dirty_mutex); + LaneWakeLocked(lane); + ThreadUnlock(&dirty_mutex); +} + +/** Starts the lane's thread unless running. Called by the main thread. */ +static bool LaneStart(WatcherLane *lane) +{ + assert(lane != NULL); + + if (lane->thread != NULL) + { + return true; + } + StoppableThread *thread = StoppableThreadStart(WatcherLaneMain, lane); + if (thread == NULL) + { + Log(LOG_LEVEL_ERR, "Unable to start the cf-reactor watcher thread of the %s watchers", + lane->name); + return false; + } + ThreadLock(&dirty_mutex); + lane->thread = thread; + ThreadUnlock(&dirty_mutex); + Log(LOG_LEVEL_VERBOSE, "Started the cf-reactor watcher thread of the %s watchers", lane->name); + return true; +} + +/** Stops the lane's thread, if running. Returns false if it didn't exit in + * time, and may still use the registry. Called by the main thread. */ +static bool LaneStop(WatcherLane *lane) +{ + assert(lane != NULL); + + ThreadLock(&dirty_mutex); + StoppableThread *thread = lane->thread; + lane->thread = NULL; + ThreadUnlock(&dirty_mutex); + + if (thread == NULL) + { + return true; + } + if (!StoppableThreadStop(thread, WATCHER_THREAD_EXIT_TIMEOUT_SECS)) + { + Log(LOG_LEVEL_ERR, + "The cf-reactor watcher thread of the %s watchers is stuck in a check, leaving it behind", + lane->name); + /* Its handle is leaked on purpose, see StoppableThreadStop(): kept + * reachable, so a leak checker doesn't report it */ + lane->stuck_thread = thread; + return false; + } + return true; +} + +static StateSlot *SlotNew(const char *hash, WatcherLane *lane, const WatcherOptions *opt) +{ + assert(hash != NULL); + assert(lane != NULL); + assert(opt != NULL); + + StateSlot *slot = xcalloc(1, sizeof(StateSlot)); + slot->hash = xstrdup(hash); + slot->lane = lane; + slot->check_callback = opt->check_callback; + slot->destroy_state = opt->destroy_state; + slot->state = opt->state; + slot->interval = opt->interval; + slot->refs = 1; + slot->next_due = 0; /* due immediately on its lane's first pass */ + slot->last_check = WATCHER_CHECK_NO_EVENT; + ThrottleInit(&slot->throttle, MonotonicClockNow()); + return slot; +} + +/** Drops a reference to the slot, freeing it with its state if it was the + * last. Holding watchers_mutex (or with no lanes). */ +static void SlotRelease(StateSlot *slot) +{ + assert(slot != NULL); + assert(slot->refs > 0); + + slot->refs--; + if (slot->refs == 0) + { + slot->destroy_state(slot->state); + free(slot->hash); + free(slot); + } +} + +/** The lane of a new watcher: the remote one if `remote`, started with the + * first remote watcher once the lanes run (else by EventWatcherInitialize()), + * falling back to the local one. Called by the main thread. */ +static WatcherLane *LaneForNewWatcher(bool remote) +{ + if (remote && (local_lane.thread == NULL || LaneStart(&remote_lane))) + { + return &remote_lane; + } + return &local_lane; +} + +// Expects interval to be strictly greater than 0, otherwise its lane will busy spin bool WatcherRegister(const Promise *pp, WatcherOptions *opt) { assert(pp != NULL); @@ -131,78 +572,315 @@ bool WatcherRegister(const Promise *pp, WatcherOptions *opt) char promise_hash[CF_HOSTKEY_STRING_SIZE]; WatcherPromiseHash(pp, promise_hash, sizeof(promise_hash)); + /* Only the main thread changes the maps, so they don't change until the + * watcher is inserted below */ + /* From the path and the mount table alone, as anything touching the file + * system of the path could block on it. For a duplicate, the lane is the + * one already started for its identical watcher. */ + const bool remote = WatchTargetIsRemote(&opt->target); + WatcherLane *lane = LaneForNewWatcher(remote); + + /* The slot of an identical watcher of the previous policy, whatever it + * is doing, see the invariants above watchers_mutex */ + StateSlot *slot = NULL; ThreadLock(&watchers_mutex); - - if (MapHasKey(watchers, promise_hash)) + const bool duplicate = MapHasKey(watchers, promise_hash); + const Watcher *prev = (old_watchers == NULL) ? NULL : MapGet(old_watchers, promise_hash); + if (!duplicate && prev != NULL && prev->slot->lane == lane) { - // ignore duplicates - ThreadUnlock(&watchers_mutex); - opt->destroy_state(opt->state); - return true; + slot = prev->slot; + slot->refs++; + /* Its wakeups were dropped while paused */ + slot->recheck = true; } + ThreadUnlock(&watchers_mutex); - // Take over the state of an identical watcher - Watcher *prev = (old_watchers != NULL) ? MapGet(old_watchers, promise_hash) : NULL; - const bool taken_over = (prev != NULL); - void *state = opt->state; - if (taken_over) + if (duplicate) { - state = prev->state; - prev->state = NULL; /* so not destroyed with prev */ + opt->destroy_state(opt->state); + return true; } - ThreadUnlock(&watchers_mutex); + const uint64_t id = (++last_watcher_number << 1) | + ((lane == &remote_lane) ? WATCHER_ID_REMOTE_BIT : 0); - if (taken_over) + if (slot != NULL) { - Log(LOG_LEVEL_DEBUG, "Reactor watcher '%s' takes over the state of an identical watcher", pp->promiser); + Log(LOG_LEVEL_DEBUG, "Reactor watcher '%s' keeps the state of the identical watcher it replaces", + pp->promiser); opt->destroy_state(opt->state); } else { - /* Without holding watchers_mutex, as the check may block (e.g. - * lstat() on an unresponsive NFS mount). The watcher isn't registered - * yet, so the watcher thread can't check it concurrently. */ - if (opt->check_callback(state) == WATCHER_CHECK_ERROR) + if (!remote) { - /* No reference for the next checks, so not registered */ - opt->destroy_state(state); - return false; + /* Without holding watchers_mutex, as the check may block (e.g. on + * a big file to hash). Not registered yet, so its lane can't check + * it concurrently. If it fails, the watcher's first successful + * check becomes the reference instead. That is how a remote + * watcher gets its reference, on its lane: here, an unresponsive + * server would block the policy (re-)read. A change before that + * first check is missed. */ + opt->check_callback(opt->state); } + slot = SlotNew(promise_hash, lane, opt); } - Watcher *w = xmalloc(sizeof(Watcher)); - w->promise_hash = xstrdup(promise_hash); + /* Before the watcher is in the registry: a wakeup from the method in + * between is for an unknown id, and ignored, but the watcher is checked on + * its lane's first pass anyway. */ + const char *method = WatchBackendsAdd(id, &opt->target, opt->methods, pp->promiser); + if (method == NULL) + { + Log(LOG_LEVEL_ERR, + "Reactor events promise '%s': none of its wakeup sources is available on this host", + pp->promiser); + ThreadLock(&watchers_mutex); + SlotRelease(slot); + ThreadUnlock(&watchers_mutex); + return false; + } + Log(LOG_LEVEL_VERBOSE, "Reactor watcher '%s' uses wakeup source '%s'%s", pp->promiser, method, + (lane == &remote_lane) ? ", on a remote file system" : ""); + + Watcher *w = xcalloc(1, sizeof(Watcher)); + w->slot = slot; + w->id = id; + w->method = method; w->promise = PromiseCopy(pp); - w->poll_interval_secs = opt->interval; - w->next_due = 0; /* due immediately on the watcher thread's first pass */ - w->last_check = WATCHER_CHECK_NO_EVENT; - w->check_callback = opt->check_callback; - w->destroy_state = opt->destroy_state; - w->state = state; ThreadLock(&watchers_mutex); - MapInsert(watchers, w->promise_hash, w); + MapInsert(watchers, slot->hash, w); ThreadUnlock(&watchers_mutex); return true; } +/* Destroy function of the registry's maps. Called by the main thread, + * holding watchers_mutex (or with no lanes). */ static void WatcherDestroy(void *item) { Watcher *w = item; - // w->state is NULL when it was borrowed by a new watcher - if (w->destroy_state != NULL && w->state != NULL) - { - w->destroy_state(w->state); - } + assert(w != NULL); + + WatchBackendsRemove(w->id, w->method); PromiseDestroy(w->promise); - free(w->promise_hash); + SlotRelease(w->slot); free(w); } -static void WatcherThreadMain(StoppableThread *thread, ARG_UNUSED void *unused) +/** The registered watcher using the slot, NULL if none or paused. Holding + * watchers_mutex. */ +static Watcher *SlotRegisteredWatcher(const StateSlot *slot) +{ + assert(slot != NULL); + + if (paused) + { + return NULL; + } + Watcher *w = MapGet(watchers, slot->hash); + return (w != NULL && w->slot == slot) ? w : NULL; +} + +/** Called by a lane, holding watchers_mutex */ +static void QueueEvent(Watcher *w, WatcherCheckResult check) +{ + assert(w != NULL); + + WatcherEvent *event = xmalloc(sizeof(WatcherEvent)); + event->watcher = w; + event->check = check; + event->woken_by = (check == WATCHER_CHECK_EVENT) ? w->slot->event_woken_by : NULL; + ThreadedQueuePush(event_queue, event); +} + +/** Waits for the burst of changes that marked watchers of the lane to settle, + * see WATCH_SETTLE_MSECS. Returns the time after the wait. */ +static time_t SettleIfMarked(const WatcherLane *lane, time_t now) +{ + assert(lane != NULL); + + ThreadLock(&dirty_mutex); + const bool marked = (lane->dirty_ids.count > 0 || lane->all_dirty); + ThreadUnlock(&dirty_mutex); + if (!marked) + { + return now; + } + + const struct timespec settle = { 0, WATCH_SETTLE_MSECS * 1000000L }; + nanosleep(&settle, NULL); + return MonotonicClockNow(); +} + +/** Takes the dirty marks of the lane into `taken`, see WatcherMarkDirty(), + * replacing the ids taken last time. Returns whether all its watchers are + * marked. */ +static bool TakeDirtyMarks(WatcherLane *lane, IdSet *taken) +{ + assert(lane != NULL); + assert(taken != NULL); + + ThreadLock(&dirty_mutex); + const IdSet previous = *taken; + *taken = lane->dirty_ids; + lane->dirty_ids = previous; + IdSetClear(&lane->dirty_ids); + const bool check_all = lane->all_dirty; + lane->all_dirty = false; + ThreadUnlock(&dirty_mutex); + return check_all; +} + +/** Whether a check of the slot is of use now, see WatcherWantsCheck(). + * While an event is held, checking again can't change what runs next, and a + * check may be costly (file_content_modified hashes the file). */ +static bool SlotCanCheck(const StateSlot *slot, time_t now) +{ + assert(slot != NULL); + + return !slot->event_held || + (!slot->event_pending && ThrottleDelayLeft(&slot->throttle, now) == 0); +} + +/** Whether to check the slot in this pass: it is due (or its watcher was woken + * up by its wakeup source) and the check is of use, see below. Called by its + * lane, holding watchers_mutex. */ +static bool WatcherWantsCheck(StateSlot *slot, time_t now, bool woken) +{ + assert(slot != NULL); + + const bool due = (now >= slot->next_due || woken); + + /* Skip the check while an event is held and check once when it can run, + * so a busy file is checked about as often as the bundles run. */ + const bool can_run = SlotCanCheck(slot, now); + if (due && !can_run) + { + slot->recheck = true; + slot->next_due = now + slot->interval; + return false; + } + return can_run && (due || slot->recheck); +} + +/** Applies the result of a check of the slot, which ran without holding + * watchers_mutex. An event is held, to be queued for the slot's registered + * watcher `w` (see WatcherQueueHeldEvent()), and an error queued for it right + * away, unless `w` is NULL: then it is reported by a later check. Returns + * whether an error was queued. Called by its lane, holding watchers_mutex. */ +static bool SlotApplyCheck(StateSlot *slot, Watcher *w, WatcherCheckResult check, time_t now) +{ + assert(slot != NULL); + + slot->next_due = now + ((check == WATCHER_CHECK_ERROR) ? + MIN(slot->interval, CHECK_ERROR_RETRY_SECS) : + slot->interval); + + if (check == WATCHER_CHECK_EVENT) + { + slot->event_held = true; + } + /* Only report a failing watcher once, not on every check, + * until it recovers */ + else if (check == WATCHER_CHECK_ERROR && slot->last_check != WATCHER_CHECK_ERROR) + { + if (w == NULL) + { + return false; + } + QueueEvent(w, WATCHER_CHECK_ERROR); + slot->last_check = check; + return true; + } + slot->last_check = check; + return false; +} + +/** Queues the event held by the slot of the registered watcher if the trigger + * limit allows. Returns whether it did. Called by its lane, holding + * watchers_mutex. */ +static bool WatcherQueueHeldEvent(Watcher *w, time_t now) +{ + assert(w != NULL); + + StateSlot *slot = w->slot; + if (slot->event_held && !slot->event_pending && + ThrottleMayRun(&slot->throttle, w->promise->promiser, now)) + { + slot->event_held = false; + slot->event_pending = true; + QueueEvent(w, WATCHER_CHECK_EVENT); + return true; + } + return false; +} + +/** How long until the slot needs its lane again. A slot waiting for its + * event to be handled is woken by EventHandled(). */ +static time_t SlotSecsUntilDue(const StateSlot *slot, time_t now) +{ + assert(slot != NULL); + + if (slot->recheck && SlotCanCheck(slot, now)) + { + return 0; + } + + time_t until_due = (slot->next_due > now) ? (slot->next_due - now) : 0; + const time_t delay_left = ThrottleDelayLeft(&slot->throttle, now); + if (slot->event_held && delay_left > 0) + { + until_due = MIN(until_due, delay_left); + } + return until_due; +} + +/** Pins the slots of the lane's watchers to check in this pass, see + * WatcherWantsCheck(), into lane->pinned. Returns how many. Holding + * watchers_mutex. */ +static size_t LanePinChecks(WatcherLane *lane, bool check_all, time_t now) +{ + assert(lane != NULL); + + size_t count = 0; + MapIterator it = MapIteratorInit(watchers); + MapKeyValue *item; + while ((item = MapIteratorNext(&it)) != NULL) + { + Watcher *w = item->value; + StateSlot *slot = w->slot; + const bool woken = (check_all || IdSetContains(&lane->taken_ids, w->id)); + if (slot->lane != lane || !WatcherWantsCheck(slot, now, woken)) + { + continue; + } + if (count == lane->pinned_capacity) + { + lane->pinned_capacity = (lane->pinned_capacity == 0) ? 16 : lane->pinned_capacity * 2; + lane->pinned = xrealloc(lane->pinned, lane->pinned_capacity * sizeof(PinnedCheck)); + } + slot->recheck = false; + slot->refs++; + lane->pinned[count].slot = slot; + /* A held event keeps the wakeup that led to it */ + if (!slot->event_held) + { + slot->event_woken_by = woken ? w->method : NULL; + } + count++; + } + return count; +} + +/** The loop of a lane: checks the lane's watchers when due or marked, and + * queues their events. Every lane runs it, each on its own watchers only. */ +static void WatcherLaneMain(StoppableThread *thread, void *arg) { + WatcherLane *lane = arg; + assert(lane != NULL); + /* Keep termination signals landing on the main thread (which owns the * daemon's HandleSignalsForDaemon()-based shutdown), never on this one. * No-op on Windows -- see signal_lib.h. */ @@ -212,46 +890,53 @@ static void WatcherThreadMain(StoppableThread *thread, ARG_UNUSED void *unused) while (!IsPendingTermination() && !StoppableThreadShouldStop(thread)) { - time_t now = time(NULL); time_t sleep_for = MAX_WATCHER_THREAD_SLEEP_SECS; bool any_event = false; + size_t lane_watchers = 0; + + const time_t now = SettleIfMarked(lane, MonotonicClockNow()); + + ThreadLock(&watchers_mutex); + /* Marks taken while paused are dropped, see `recheck` */ + const bool check_all = TakeDirtyMarks(lane, &lane->taken_ids); + const size_t pinned_count = paused ? 0 : LanePinChecks(lane, check_all, now); + ThreadUnlock(&watchers_mutex); + + /* Check them, without the lock: the checks may block */ + for (size_t i = 0; i < pinned_count; i++) + { + StateSlot *slot = lane->pinned[i].slot; + lane->pinned[i].check = slot->check_callback(slot->state); + } ThreadLock(&watchers_mutex); - /* Nothing to check until EventWatcherWakeUp() or EventWatcherResume() - * (EventWatcherPause() empties `watchers`) wakes the thread up */ - const bool idle = (MapSize(watchers) == 0); + /* Apply the results and unpin. The watchers may have been replaced + * meanwhile, by a policy re-read. */ + for (size_t i = 0; i < pinned_count; i++) + { + StateSlot *slot = lane->pinned[i].slot; + any_event |= SlotApplyCheck(slot, SlotRegisteredWatcher(slot), lane->pinned[i].check, + now); + SlotRelease(slot); + } + /* Queue the held events */ MapIterator it = MapIteratorInit(watchers); MapKeyValue *item; - while (!paused && (item = MapIteratorNext(&it)) != NULL) + while ((item = MapIteratorNext(&it)) != NULL) { Watcher *w = item->value; - - // check_callback cannot be null if it is correctly assigned during policy evaluation - assert(w->check_callback != NULL); - - if (now >= w->next_due) + if (w->slot->lane != lane) { - WatcherCheckResult check = w->check_callback(w->state); - w->next_due = now + w->poll_interval_secs; - - /* Only report a failing watcher once, not on every check, - * until it recovers */ - if (check == WATCHER_CHECK_EVENT || - (check == WATCHER_CHECK_ERROR && w->last_check != WATCHER_CHECK_ERROR)) - { - WatcherEvent *event = xmalloc(sizeof(WatcherEvent)); - event->watcher = w; - event->check = check; - ThreadedQueuePush(event_queue, event); - any_event = true; - } - w->last_check = check; + continue; + } + lane_watchers++; + if (!paused) + { + any_event |= WatcherQueueHeldEvent(w, now); + sleep_for = MIN(sleep_for, SlotSecsUntilDue(w->slot, now)); } - - time_t until_due = (w->next_due > now) ? (w->next_due - now) : 0; - sleep_for = MIN(sleep_for, until_due); } ThreadUnlock(&watchers_mutex); @@ -261,7 +946,11 @@ static void WatcherThreadMain(StoppableThread *thread, ARG_UNUSED void *unused) WakeupChannelNotify(&wakeup_channel); } - if (idle) + /* Wakeup sources, new watchers and shutdown wake the thread up, and + * the time of everything else is in sleep_for. Without watchers, + * nothing to check until EventWatcherWakeUp() or EventWatcherResume() + * (EventWatcherPause() empties `watchers`). */ + if (lane_watchers == 0) { StoppableThreadSleepUntilWoken(thread); } @@ -270,6 +959,27 @@ static void WatcherThreadMain(StoppableThread *thread, ARG_UNUSED void *unused) StoppableThreadSleep(thread, sleep_for); } } + free(lane->pinned); + lane->pinned = NULL; + lane->pinned_capacity = 0; + IdSetDestroy(&lane->taken_ids); +} + +/** Whether a registered watcher is on the remote lane. Called by the main + * thread before the lanes run. */ +static bool AnyRemoteWatcher(void) +{ + MapIterator it = MapIteratorInit(watchers); + MapKeyValue *item; + while ((item = MapIteratorNext(&it)) != NULL) + { + const Watcher *w = item->value; + if (w->slot->lane == &remote_lane) + { + return true; + } + } + return false; } bool EventWatcherInitialize(int *fd) @@ -284,11 +994,13 @@ bool EventWatcherInitialize(int *fd) } event_queue = ThreadedQueueNew(16, free); + WatchBackendsInitialize(&WATCH_HOST); - watcher_thread = StoppableThreadStart(WatcherThreadMain, NULL); - if (watcher_thread == NULL) + /* The remote lane too if remote watchers were registered before */ + if (!LaneStart(&local_lane) || (AnyRemoteWatcher() && !LaneStart(&remote_lane))) { - Log(LOG_LEVEL_ERR, "Unable to start cf-reactor watcher thread"); + LaneStop(&local_lane); + WatchBackendsFinalize(); ThreadedQueueDestroy(event_queue); event_queue = NULL; WakeupChannelClose(&wakeup_channel); @@ -303,10 +1015,31 @@ bool EventWatcherInitialize(int *fd) return true; } -/* Watchers are only destroyed by the main thread (EventWatcherResume()), - * which is also the one handling events, and never while events are queued, - * so the watchers of queued events can be used without holding - * watchers_mutex. Their promise is never modified by the watcher thread. */ +/** The event's bundles have run: the watcher may queue events again, and what + * it observed meanwhile (event_held), including the bundles' own changes, + * fires now. A converging bundle changes nothing the second time, so this + * ends. Taking the state after the run as the reference would avoid that run, + * but lose changes made by anything else meanwhile. */ +static void EventHandled(Watcher *w) +{ + assert(w != NULL); + + ThreadLock(&watchers_mutex); + w->slot->event_pending = false; + const bool held = w->slot->event_held; + ThreadUnlock(&watchers_mutex); + + if (held) + { + LaneWake(w->slot->lane); + } +} + +/* Watchers are only removed by the main thread (EventWatcherPause(), + * EventWatcherResume()), which is also the one handling events, and never + * while events are queued, so the watchers of queued events can be used + * without holding watchers_mutex. Their promise is never modified by the + * lanes. */ static void HandleQueuedEvents(EvalContext *ctx, WatcherEventFn on_event) { assert(on_event != NULL); @@ -315,16 +1048,29 @@ static void HandleQueuedEvents(EvalContext *ctx, WatcherEventFn on_event) while (ThreadedQueuePop(event_queue, &item, 0)) { WatcherEvent *event = item; - const Watcher *w = event->watcher; + Watcher *w = event->watcher; if (event->check == WATCHER_CHECK_ERROR) { Log(LOG_LEVEL_ERR, "Reactor watcher '%s' failed to check for its event", w->promise->promiser); } else { - Log(LOG_LEVEL_VERBOSE, "Reactor watcher '%s' fired", w->promise->promiser); + if (event->woken_by != NULL) + { + Log(LOG_LEVEL_VERBOSE, "Reactor watcher '%s' fired, woken by its wakeup source '%s'", + w->promise->promiser, event->woken_by); + } + else + { + Log(LOG_LEVEL_VERBOSE, "Reactor watcher '%s' fired, at its interval", + w->promise->promiser); + } } on_event(ctx, w->promise, event->check); + if (event->check == WATCHER_CHECK_EVENT) + { + EventHandled(w); + } free(event); } } @@ -350,7 +1096,7 @@ void EventWatcherPause(EvalContext *ctx, WatcherEventFn on_event) { assert(on_event != NULL); - /* A pass of the watcher thread holds watchers_mutex, so once `paused` is + /* A pass of a lane holds watchers_mutex, so once `paused` is * set no further events are queued */ ThreadLock(&watchers_mutex); paused = true; @@ -363,14 +1109,14 @@ void EventWatcherPause(EvalContext *ctx, WatcherEventFn on_event) ThreadLock(&watchers_mutex); - /* Paused, so the watcher thread can't have queued any event since */ + /* Paused, so the lanes can't have queued any event since */ if (event_queue != NULL && !ThreadedQueueIsEmpty(event_queue)) { ProgrammingError("Events of reactor watchers still queued after handling them while pausing"); } /* Set the watchers aside before the policy is re-read, for the watchers - * registered from it to take over the state of the identical ones, see + * registered from it to share the state slots of the identical ones, see * WatcherRegister(). If paused twice in a row, the watchers of the last * policy are the ones registered since the first pause. */ MapDestroy(old_watchers); @@ -383,7 +1129,8 @@ void EventWatcherPause(EvalContext *ctx, WatcherEventFn on_event) void EventWatcherResume(void) { ThreadLock(&watchers_mutex); - /* The watchers of the previous policy not taken over */ + /* The watchers of the previous policy, their slots go with the last of + * their references */ MapDestroy(old_watchers); old_watchers = NULL; paused = false; @@ -392,31 +1139,110 @@ void EventWatcherResume(void) EventWatcherWakeUp(); } +static int CompareReportEntries(const void *a, const void *b, ARG_UNUSED void *user_data) +{ + const JsonElement *entry_a = a; + const JsonElement *entry_b = b; + const int by_bundle = StringSafeCompare(JsonObjectGetAsString(entry_a, REACTOR_WATCHERS_KEY_BUNDLE), + JsonObjectGetAsString(entry_b, REACTOR_WATCHERS_KEY_BUNDLE)); + if (by_bundle != 0) + { + return by_bundle; + } + return StringSafeCompare(JsonObjectGetAsString(entry_a, REACTOR_WATCHERS_KEY_PROMISER), + JsonObjectGetAsString(entry_b, REACTOR_WATCHERS_KEY_PROMISER)); +} + +/** Writes the report of the method each watcher uses, for Enterprise + * reporting, sorted by bundle then promiser, so that the same watchers give + * the same report (see WatchersReportWrite()) whatever the map's order */ +static void WriteWatchersReport(void) +{ + Seq *entries = SeqNew(16, NULL); + + ThreadLock(&watchers_mutex); + MapIterator it = MapIteratorInit(watchers); + MapKeyValue *item; + while ((item = MapIteratorNext(&it)) != NULL) + { + const Watcher *w = item->value; + JsonElement *entry = JsonObjectCreate(4); + JsonObjectAppendString(entry, REACTOR_WATCHERS_KEY_PROMISER, w->promise->promiser); + const Bundle *bundle = PromiseGetBundle(w->promise); + if (StringEqual(bundle->ns, NamespaceDefault())) + { + JsonObjectAppendString(entry, REACTOR_WATCHERS_KEY_BUNDLE, bundle->name); + } + else + { + /* So same-named bundles of different namespaces don't collide */ + char *qualified = StringFormat("%s:%s", bundle->ns, bundle->name); + JsonObjectAppendString(entry, REACTOR_WATCHERS_KEY_BUNDLE, qualified); + free(qualified); + } + JsonObjectAppendString(entry, REACTOR_WATCHERS_KEY_WAKEUP_SOURCE, w->method); + JsonObjectAppendInteger64(entry, REACTOR_WATCHERS_KEY_INTERVAL, + (int64_t) w->slot->interval); + SeqAppend(entries, entry); + } + ThreadUnlock(&watchers_mutex); + + SeqSort(entries, CompareReportEntries, NULL); + const size_t count = SeqLength(entries); + JsonElement *report = JsonArrayCreate(count); + for (size_t i = 0; i < count; i++) + { + JsonArrayAppendObject(report, SeqAt(entries, i)); + } + SeqDestroy(entries); + + WatchersReportWrite(report); + JsonDestroy(report); +} + void EventWatcherWakeUp(void) { /* Called once the watchers of the (re-)read policy are registered */ ThreadLock(&watchers_mutex); const size_t count = MapSize(watchers); ThreadUnlock(&watchers_mutex); + WriteWatchersReport(); Log(LOG_LEVEL_VERBOSE, "cf-reactor event watcher has %zu watcher(s)", count); - if (watcher_thread != NULL) - { - StoppableThreadWake(watcher_thread); - } + /* Until the next re-read */ + WatchLocationForgetMountTable(); + + LaneWake(&local_lane); + LaneWake(&remote_lane); } void EventWatcherFinalize(void) { - bool joined = StoppableThreadStop(watcher_thread, WATCHER_THREAD_EXIT_TIMEOUT_SECS); - watcher_thread = NULL; - if (!joined) + /* Clean shutdown: the watchers are gone, so is their report */ + WatchersReportRemove(); + + /* Both, even if the first is stuck. The local lane first: the remote one + * is the likely one to be stuck, on a server that doesn't respond. */ + const bool local_joined = LaneStop(&local_lane); + const bool remote_joined = LaneStop(&remote_lane); + + /* Once the lanes are stopped, so nothing wakes them up any more. A lane + * left behind doesn't call the methods (only the main thread does). */ + WatchBackendsFinalize(); + + WatchLocationForgetMountTable(); + + if (!local_joined || !remote_joined) { - /* The thread may still be running and using the event queue, the - * wakeup channel and the watcher registry, so leak them rather than - * risk a use-after-free. We're shutting down anyway. */ + /* A lane may still be running and using the event queue, the wakeup + * channel, the watcher registry and the states, so leak them rather + * than risk a use-after-free. We're shutting down anyway. */ return; } + IdSetDestroy(&local_lane.dirty_ids); + local_lane.all_dirty = false; + IdSetDestroy(&remote_lane.dirty_ids); + remote_lane.all_dirty = false; ThreadedQueueDestroy(event_queue); event_queue = NULL; @@ -424,3 +1250,26 @@ void EventWatcherFinalize(void) WatcherRegistryFinalize(); } + +void WatcherMarkDirty(uint64_t watcher_id) +{ + /* Only wakes the lane of the watcher up, so a busy local file doesn't + * make the remote lane settle and pass for nothing */ + WatcherLane *lane = LaneOfId(watcher_id); + ThreadLock(&dirty_mutex); + IdSetAdd(&lane->dirty_ids, watcher_id); + LaneWakeLocked(lane); + ThreadUnlock(&dirty_mutex); +} + +void WatcherMarkAllDirty(void) +{ + WatcherLane *const lanes[] = { &local_lane, &remote_lane }; + ThreadLock(&dirty_mutex); + for (size_t i = 0; i < sizeof(lanes) / sizeof(lanes[0]); i++) + { + lanes[i]->all_dirty = true; + LaneWakeLocked(lanes[i]); + } + ThreadUnlock(&dirty_mutex); +} diff --git a/cf-reactor/watcher.h b/cf-reactor/watcher.h index 46f1ef6c137..3fd169485ca 100644 --- a/cf-reactor/watcher.h +++ b/cf-reactor/watcher.h @@ -27,6 +27,7 @@ #include /* Bundle */ #include /* CF_HOSTKEY_STRING_SIZE */ +#include typedef enum { @@ -52,6 +53,8 @@ typedef struct WatcherStateDestroyFn destroy_state; void *state; /* owned by the registry once registered */ time_t interval; /* seconds between two checks, > 0 */ + ReactorWatchTarget target; /* what the event is about, for the wakeup sources */ + const Rlist *methods; /* 'wakeup_sources', NULL for the default */ } WatcherOptions; /** @@ -88,11 +91,20 @@ void WatcherRegistryFinalize(void); * ignored, so that the 'then' bundle isn't run twice for the same event. * * If a watcher of the previous policy (see EventWatcherPause()) was - * registered for an identical expanded promise, the new watcher takes over - * its state instead of opt->state, so that the events happening while paused - * are detected. Otherwise opt->state is checked once right away, as the - * reference for the next checks, so that only what happens from now on - * fires. If that check fails, the watcher is not registered. + * registered for an identical expanded promise, the new watcher shares its + * state instead of using opt->state, so that the events happening while + * paused are detected, with any event it observed but didn't run yet, and its + * limit on how often the 'then' bundles run. This doesn't wait for a check of + * that state still running (it may block): the new watcher's checks follow + * it. Unless the watched file moved to or from a remote file system (see + * WatchTargetIsRemote()), as the state is then about another file system: + * the new watcher starts afresh. Otherwise opt->state is checked once right + * away, as the reference for the next checks, so that only what happens from + * now on fires. If that check fails, the watcher's first successful check + * becomes the reference instead. So it does for a watcher of a file on a + * remote file system, which isn't checked here, as that could block for as + * long as its server doesn't respond: its checks run on a thread of their + * own. * * @param pp the expanded events promise. The watcher keeps a copy of it * (see PromiseCopy()), passed back to the WatcherEventFn on event. @@ -102,8 +114,13 @@ void WatcherRegistryFinalize(void); * owned by the registry, which destroys it right away if not * used, so it must not be used by the caller afterwards. * opt->interval must be strictly greater than 0. - * @return false if the check of opt->state failed (WATCHER_CHECK_ERROR), in - * which case the watcher is not registered, true otherwise. + * + * The watcher is also watched with the first available method of + * opt->methods (see WatchBackendsAdd()), which wakes it up to be checked + * before its interval elapses. + * + * @return false if none of opt->methods is available and they don't include + * "poll", in which case the watcher is not registered, true otherwise. */ bool WatcherRegister(const Promise *pp, WatcherOptions *opt); bool EventWatcherInitialize(int *fd); @@ -130,8 +147,8 @@ void EventWatcherPause(EvalContext *ctx, WatcherEventFn on_event); void EventWatcherResume(void); /** - * @brief Wake the watcher thread up to check the registered watchers - * immediately. Without watchers, the watcher thread sleeps until woken up, so + * @brief Wake the watcher threads up to check the registered watchers + * immediately. Without watchers, a watcher thread sleeps until woken up, so * this must be called after registering watchers outside of an * EventWatcherPause()/EventWatcherResume() pair (which wakes it up itself), * i.e. after the watchers of the initial policy. Logs the number of @@ -140,4 +157,14 @@ void EventWatcherResume(void); void EventWatcherWakeUp(void); void EventWatcherFinalize(void); +/** + * @brief Check the watcher with this id as soon as possible, regardless of + * its interval. Thread-safe, an id that isn't registered is ignored. + * See ReactorWatchHost. + */ +void WatcherMarkDirty(uint64_t watcher_id); + +/** @brief Check all the watchers as soon as possible. Thread-safe. */ +void WatcherMarkAllDirty(void); + #endif diff --git a/cf-reactor/watcher_report.c b/cf-reactor/watcher_report.c new file mode 100644 index 00000000000..a4ff4ce5bd5 --- /dev/null +++ b/cf-reactor/watcher_report.c @@ -0,0 +1,126 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#include +#include // REACTOR_WATCHERS_REPORT_FILE +#include +#include +#include // safe_fopen() +#include // StringWriter() +#include // StringEqual() +#include // GetStateDir() + +/* What the report file was last written with. Only used by the thread that + * writes the reports (the main thread). */ +static char *last_written = NULL; + +static bool ReportPaths(char *path, size_t path_size, char *tmp_path, size_t tmp_path_size) +{ + int ret = snprintf(path, path_size, "%s%c%s", + GetStateDir(), FILE_SEPARATOR, REACTOR_WATCHERS_REPORT_FILE); + if (ret < 0 || (size_t) ret >= path_size) + { + return false; + } + ret = snprintf(tmp_path, tmp_path_size, "%s.tmp", path); + return (ret >= 0 && (size_t) ret < tmp_path_size); +} + +void WatchersReportWrite(const JsonElement *report) +{ + assert(report != NULL); + if (report == NULL) + { + return; + } + + Writer *writer = StringWriter(); + JsonWrite(writer, report, 0); + char *content = StringWriterClose(writer); + + /* Rewriting it with the same content would still change its mtime, which + * the reader takes for the time the watchers were updated */ + if (last_written != NULL && StringEqual(last_written, content)) + { + free(content); + return; + } + + char path[PATH_MAX]; + char tmp_path[PATH_MAX]; + if (!ReportPaths(path, sizeof(path), tmp_path, sizeof(tmp_path))) + { + Log(LOG_LEVEL_VERBOSE, "The path of the reactor watchers report is too long"); + free(content); + return; + } + + FILE *file = safe_fopen(tmp_path, "w"); + if (file == NULL) + { + Log(LOG_LEVEL_VERBOSE, "Failed to write the reactor watchers report '%s': %s", + tmp_path, GetErrorStr()); + free(content); + return; + } + + const size_t length = strlen(content); + const bool written = (fwrite(content, 1, length, file) == length); + const bool closed = (fclose(file) == 0); + if (!written || !closed) + { + Log(LOG_LEVEL_VERBOSE, "Failed to write the reactor watchers report '%s': %s", + tmp_path, GetErrorStr()); + unlink(tmp_path); + free(content); + return; + } + + if (rename(tmp_path, path) != 0) + { + Log(LOG_LEVEL_VERBOSE, "Failed to rename '%s' to '%s': %s", + tmp_path, path, GetErrorStr()); + unlink(tmp_path); + free(content); + return; + } + + free(last_written); + last_written = content; +} + +void WatchersReportRemove(void) +{ + char path[PATH_MAX]; + char tmp_path[PATH_MAX]; + if (ReportPaths(path, sizeof(path), tmp_path, sizeof(tmp_path)) && + unlink(path) != 0 && errno != ENOENT) + { + Log(LOG_LEVEL_VERBOSE, "Failed to remove the reactor watchers report '%s': %s", + path, GetErrorStr()); + } + + free(last_written); + last_written = NULL; +} diff --git a/cf-reactor/watcher_report.h b/cf-reactor/watcher_report.h new file mode 100644 index 00000000000..f478040d0b5 --- /dev/null +++ b/cf-reactor/watcher_report.h @@ -0,0 +1,47 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef CFENGINE_WATCHER_REPORT_H +#define CFENGINE_WATCHER_REPORT_H + +#include + +/** + * @brief Writes the report of the watchers to REACTOR_WATCHERS_REPORT_FILE in + * the state directory, for Enterprise reporting. + * + * Written aside and renamed over, so that a reader never sees half of it, and + * only if it differs from the last one written, to keep the file's mtime (the + * reader's "updated at") for when the watchers changed. Failures are logged, + * not returned: the report is best effort, and a failed write keeps the old one. + */ +void WatchersReportWrite(const JsonElement *report); + +/** + * @brief Removes the report, so that the watchers of a stopped cf-reactor + * don't linger in reporting. + */ +void WatchersReportRemove(void); + +#endif // CFENGINE_WATCHER_REPORT_H diff --git a/configure.ac b/configure.ac index 414aa1d4c66..8bd05bff0df 100644 --- a/configure.ac +++ b/configure.ac @@ -192,6 +192,10 @@ dnl and LIBS are appropriately set for the local pthreads situation. dnl ###################################################################### AC_CHECK_FUNCS([pthread_cancel]) +dnl cf-reactor/stoppable_thread.c times its waits on the monotonic clock when +dnl the condition variable's clock can be set (not on macOS). +AC_CHECK_FUNCS([pthread_condattr_setclock]) + dnl ###################################################################### dnl Whether to build extensions as builtin extensions or a separate dnl plugin. The default is plugin. diff --git a/libpromises/Makefile.am b/libpromises/Makefile.am index 99368a35df4..ef68e97e181 100644 --- a/libpromises/Makefile.am +++ b/libpromises/Makefile.am @@ -150,6 +150,7 @@ libpromises_la_SOURCES = \ process_lib.h process_unix_priv.h \ promises.c promises.h \ prototypes3.h \ + reactor_watch_backend.h \ rlist.c rlist.h \ scope.c scope.h \ shared_lib.c shared_lib.h \ diff --git a/libpromises/enterprise_stubs.c b/libpromises/enterprise_stubs.c index dccf8bc2f67..cfe42d269a8 100644 --- a/libpromises/enterprise_stubs.c +++ b/libpromises/enterprise_stubs.c @@ -232,6 +232,11 @@ ENTERPRISE_VOID_FUNC_2ARG_DEFINE_STUB(void, Nova_ClassHistoryEnable, { } +ENTERPRISE_FUNC_0ARG_DEFINE_STUB(const ReactorWatchBackend *const *, ReactorEnterpriseWatchBackends) +{ + return NULL; +} + ENTERPRISE_FUNC_0ARG_DEFINE_STUB(size_t, ReactorNovaMaxFds) { return 0; diff --git a/libpromises/mod_reactor.c b/libpromises/mod_reactor.c index ea1f8c6b616..64c361c628e 100644 --- a/libpromises/mod_reactor.c +++ b/libpromises/mod_reactor.c @@ -54,13 +54,14 @@ static const ConstraintSyntax when_constraints[] = ConstraintSyntaxNewString(REACTOR_USER_ADDED, CF_ANYSTRING, "User to react for on addition", SYNTAX_STATUS_NORMAL), ConstraintSyntaxNewString(REACTOR_USER_REMOVED, CF_ANYSTRING, "User to react for on removal", SYNTAX_STATUS_NORMAL), ConstraintSyntaxNewInt("interval", REACTOR_INTERVAL_RANGE, "Seconds between two checks for the event", SYNTAX_STATUS_NORMAL), + ConstraintSyntaxNewStringList(REACTOR_WAKEUP_SOURCES, REACTOR_WAKEUP_SOURCES_RANGE, "Methods to watch for the event with, in order of preference. Without '" REACTOR_WAKEUP_SOURCE_POLL "', the promise is not kept if none of them is available", SYNTAX_STATUS_NORMAL), ConstraintSyntaxNewNull() }; /* Checks that a 'when' body has an event attribute, i.e. any attribute other - * than the global ones (meta, inherit_from) and 'interval', which only sets - * how the event is checked for. A body inheriting from another one may get - * its event from it, and that one is checked on its own. */ + * than the global ones (meta, inherit_from), 'interval' and 'wakeup_sources', + * which only set how the event is checked for. A body inheriting from another + * may get its event from it, and that one is checked on its own. */ static bool WhenBodyCheck(const Body *body, Seq *errors) { assert(body != NULL); @@ -73,7 +74,9 @@ static bool WhenBodyCheck(const Body *body, Seq *errors) for (size_t i = 0; i < SeqLength(body->conlist); i++) { const Constraint *cp = SeqAt(body->conlist, i); - if (!StringEqual(cp->lval, "meta") && !StringEqual(cp->lval, "interval")) + if (!StringEqual(cp->lval, "meta") && + !StringEqual(cp->lval, "interval") && + !StringEqual(cp->lval, REACTOR_WAKEUP_SOURCES)) { return true; } diff --git a/libpromises/mod_reactor.h b/libpromises/mod_reactor.h index eeb5380a122..f115222dd0f 100644 --- a/libpromises/mod_reactor.h +++ b/libpromises/mod_reactor.h @@ -39,6 +39,13 @@ #define REACTOR_USER_ADDED "user_added" #define REACTOR_USER_REMOVED "user_removed" +/* How the event of a 'when' body is watched for, an ordered list of methods. + * Every method name of every edition is known here, so that policy validates + * the same way everywhere; a method this binary doesn't have is skipped. */ +#define REACTOR_WAKEUP_SOURCES "wakeup_sources" +#define REACTOR_WAKEUP_SOURCE_POLL "poll" +#define REACTOR_WAKEUP_SOURCES_RANGE "poll|inotify|fanotify|ebpf|usn|rdcw" + /* Range of the 'interval' attribute of the 'when' body of events promises */ #define REACTOR_INTERVAL_RANGE "1,99999999999" diff --git a/libpromises/prototypes3.h b/libpromises/prototypes3.h index 061e7afc40d..b87e419a19c 100644 --- a/libpromises/prototypes3.h +++ b/libpromises/prototypes3.h @@ -29,6 +29,7 @@ #include #include #include +#include bool BootstrapAllowed(void); @@ -84,6 +85,9 @@ ENTERPRISE_FUNC_3ARG_DECLARE(bool, ReactorNovaInitialize, int*, fds, size_t, max ENTERPRISE_VOID_FUNC_1ARG_DECLARE(void, ReactorNovaHandleTimeout, time_t *, next_tick); ENTERPRISE_VOID_FUNC_3ARG_DECLARE(void, ReactorNovaHandleEvents, fd_set *, readfds, int *, fds, time_t *, next_tick); ENTERPRISE_VOID_FUNC_0ARG_DECLARE(void, ReactorNovaFinalize); +/* NULL-terminated array of the extra cf-reactor wakeup sources, see + * reactor_watch_backend.h, or NULL */ +ENTERPRISE_FUNC_0ARG_DECLARE(const ReactorWatchBackend *const *, ReactorEnterpriseWatchBackends); /* manual.c */ diff --git a/libpromises/reactor_watch_backend.h b/libpromises/reactor_watch_backend.h new file mode 100644 index 00000000000..dad5d63d336 --- /dev/null +++ b/libpromises/reactor_watch_backend.h @@ -0,0 +1,95 @@ +/* + Copyright 2026 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef CFENGINE_REACTOR_WATCH_BACKEND_H +#define CFENGINE_REACTOR_WATCH_BACKEND_H + +/* Interface between cf-reactor and the methods it can watch for events with + * (the 'wakeup_sources' attribute of 'when' bodies), shared with the + * Enterprise extension, which provides some of the methods. + * + * A method doesn't detect events itself. It only tells cf-reactor that + * something touched what a watcher is about, by marking the watcher dirty, + * and the watcher's own check then decides whether its event happened. So + * every event type behaves the same with every method, and polling at the + * watcher's interval keeps running underneath as the reference. */ + +#include + +/* JSON array written to the state directory by cf-reactor when its watchers + * change, and removed when it stops. One object per watcher, sorted by bundle + * then promiser: "promiser", "bundle" (qualified with the namespace unless it + * is the default one), "wakeup_source" (the one it resolved to) and + * "interval" in seconds, see the REACTOR_WATCHERS_KEY_ macros. Read by + * Enterprise reporting. */ +#define REACTOR_WATCHERS_REPORT_FILE "cf_reactor_watchers.json" +#define REACTOR_WATCHERS_KEY_PROMISER "promiser" +#define REACTOR_WATCHERS_KEY_BUNDLE "bundle" +#define REACTOR_WATCHERS_KEY_WAKEUP_SOURCE "wakeup_source" +#define REACTOR_WATCHERS_KEY_INTERVAL "interval" + +typedef enum +{ + REACTOR_WATCH_TARGET_FILE, /* the file at `path`, which may not exist */ + REACTOR_WATCH_TARGET_USERS, /* the local user accounts, `path` is NULL */ +} ReactorWatchTargetType; + +typedef struct +{ + ReactorWatchTargetType type; + const char *path; +} ReactorWatchTarget; + +/* What cf-reactor provides to the methods. Both functions are thread-safe, + * and may be called with a watcher_id that is no longer registered, which is + * ignored. */ +typedef struct +{ + void (*mark_dirty)(uint64_t watcher_id); /* check this watcher now */ + void (*mark_all_dirty)(void); /* events were lost, check all now */ +} ReactorWatchHost; + +typedef enum +{ + REACTOR_WATCH_STARTED, + REACTOR_WATCH_UNSUPPORTED, /* not on this host, never tried again */ + REACTOR_WATCH_START_FAILED, /* e.g. out of resources, tried again later */ +} ReactorWatchStartResult; + +/* A method to watch for events with. Each one runs its own thread (or uses + * the OS's own callbacks) to read its notifications, and calls the host's + * mark_dirty() for the watchers they concern. start(), stop(), add() and + * remove() are only ever called from cf-reactor's main thread, never + * concurrently with each other, and add() and remove() only between start() + * and stop(). */ +typedef struct +{ + const char *name; /* as in 'wakeup_sources' */ + ReactorWatchStartResult (*start)(const ReactorWatchHost *host); + void (*stop)(void); + bool (*add)(uint64_t watcher_id, const ReactorWatchTarget *target); + void (*remove)(uint64_t watcher_id); +} ReactorWatchBackend; + +#endif diff --git a/misc/selinux/cfengine-enterprise.te.all b/misc/selinux/cfengine-enterprise.te.all index c03df4170d3..c7d4629f828 100644 --- a/misc/selinux/cfengine-enterprise.te.all +++ b/misc/selinux/cfengine-enterprise.te.all @@ -17,6 +17,7 @@ require { attribute non_auth_file_type; type autofs_t; type bin_t; + type bpf_t; type cert_t; type devlog_t; type kernel_t; @@ -750,6 +751,16 @@ allow cfengine_reactor_t sendmail_exec_t:file map; allow cfengine_reactor_t sendmail_exec_t:file { execute execute_no_trans open read }; allow cfengine_reactor_t smtp_port_t:tcp_socket name_connect; +# cf-reactor wakeup sources. A denial only makes a watcher fall back to polling. +# The 'watch' and CAP_BPF/CAP_PERFMON rules are in the EL9/EL10 files, the EL8 +# policy doesn't define them. +# ebpf reads the kernel's BTF (/sys/kernel/btf/vmlinux), then loads and +# attaches its watch program (kprobe.multi or fentry) +allow cfengine_reactor_t sysfs_t:file { getattr open read }; +allow cfengine_reactor_t self:bpf { map_create map_read map_write prog_load prog_run }; +# libbpf opens /sys/fs/bpf to try a BPF token, which it doesn't need +dontaudit cfengine_reactor_t bpf_t:dir read; + #============= cfengine_action_script_t ============== # A special type and domain for action (notification/alert) scripts executed by diff --git a/misc/selinux/cfengine-enterprise.te.el10 b/misc/selinux/cfengine-enterprise.te.el10 index 1aee2439e41..5e51d89d1be 100644 --- a/misc/selinux/cfengine-enterprise.te.el10 +++ b/misc/selinux/cfengine-enterprise.te.el10 @@ -48,3 +48,13 @@ allow cfengine_reactor_t kernel_t:unix_stream_socket connectto; #============= cfengine_serverd_t ============== allow cfengine_serverd_t http_port_t:tcp_socket name_connect; + +#============= cf-reactor wakeup sources ============== +# Policy may ask for any method at any time (the default list tries ebpf), so these rules can't depend on policy. +# A denial only makes a watcher fall back to polling. +# inotify and fanotify watch the watched file and its parent directory (or the +# nearest existing ancestor), which can be anywhere +allow cfengine_reactor_t file_type:dir watch; +allow cfengine_reactor_t file_type:file watch; +# ebpf +allow cfengine_reactor_t self:capability2 { bpf perfmon }; diff --git a/misc/selinux/cfengine-enterprise.te.el9 b/misc/selinux/cfengine-enterprise.te.el9 index 14fe34b0ca0..54572f10958 100644 --- a/misc/selinux/cfengine-enterprise.te.el9 +++ b/misc/selinux/cfengine-enterprise.te.el9 @@ -14,3 +14,13 @@ allow cfengine_reactor_t systemd_userdbd_runtime_t:sock_file write; # these permissions are not be needed if changes from ENT-12954 to masterfiles policy move inventory from common to an agent bundle are in place. allow cfengine_serverd_t http_port_t:tcp_socket name_connect; allow cfengine_execd_t http_port_t:tcp_socket name_connect; + +#============= cf-reactor wakeup sources ============== +# Policy may ask for any method at any time (the default list tries ebpf), so these rules can't depend on policy. +# A denial only makes a watcher fall back to polling. +# inotify and fanotify watch the watched file and its parent directory (or the +# nearest existing ancestor), which can be anywhere +allow cfengine_reactor_t file_type:dir watch; +allow cfengine_reactor_t file_type:file watch; +# ebpf +allow cfengine_reactor_t self:capability2 { bpf perfmon }; diff --git a/tests/reactor/tests/00_basics/009_interval.cf b/tests/reactor/tests/00_basics/009_interval.cf index 008cf55c893..b737b141238 100644 --- a/tests/reactor/tests/00_basics/009_interval.cf +++ b/tests/reactor/tests/00_basics/009_interval.cf @@ -14,6 +14,10 @@ body when deleted(file) { file_deleted => "$(file)"; interval => 10; + + # The interval only spaces the checks of polling: a wakeup source wakes the + # watcher up on every change + wakeup_sources => { "poll" }; } bundle reactor test diff --git a/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.cf b/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.cf new file mode 100644 index 00000000000..09d0c8a1d5a --- /dev/null +++ b/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.cf @@ -0,0 +1,34 @@ +####################################################### +# +# Test that stepping the wall clock back doesn't postpone checks +# +####################################################### +body common control +{ + inputs => { "../../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when deleted(file) +{ + file_deleted => "$(file)"; + interval => 1; + + # A wakeup source wakes the watcher up whatever the clock says: only + # polling shows whether the next check is postponed + wakeup_sources => { "poll" }; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + + events: + "$(promise_name)" + when => deleted("$(test)"), + then => record_event("$(events_file)", "$(test) was deleted"); +} diff --git a/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.sh b/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.sh new file mode 100644 index 00000000000..843a33c86f9 --- /dev/null +++ b/tests/reactor/tests/00_basics/unsafe/001_clock_step_back.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# check that stepping the wall clock back while cf-reactor is running doesn't +# postpone its checks by the size of the step, which is longer than the +# harness waits for an event. Steps the system clock with sudo. + +set -ex + +. "$(dirname "$0")/../../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +STEP_SECS=300 + +touch "$TESTFILE" + +start_reactor "$POLICY_FILE" + +restore_clock() { + sudo date -s "@$(($(date +%s) + STEP_SECS))" >/dev/null + stop_reactor +} +trap restore_clock EXIT + +wait_for_log "File '$TESTFILE' is present, watching for its deletion" + +# step back while the watcher thread is between two checks +sudo date -s "@$(($(date +%s) - STEP_SECS))" >/dev/null + +rm "$TESTFILE" +wait_for_lines "$EVENTS_FILE" 1 +check_event "$EVENTS_FILE" 1 "$TESTFILE was deleted" diff --git a/tests/reactor/tests/03_wakeup_sources/003_no_method_available.cf b/tests/reactor/tests/03_wakeup_sources/003_no_method_available.cf new file mode 100644 index 00000000000..7c6cdaf2772 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/003_no_method_available.cf @@ -0,0 +1,30 @@ +####################################################### +# +# Test that without poll, an unavailable method makes the promise not kept +# +####################################################### +body common control +{ + inputs => { "../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when deleted(file) +{ + file_deleted => "$(file)"; + wakeup_sources => { "usn", "rdcw" }; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + + events: + "$(promise_name)" + when => deleted("$(test)"), + then => record_event("$(events_file)", "$(test) was deleted"); +} diff --git a/tests/reactor/tests/03_wakeup_sources/003_no_method_available.sh b/tests/reactor/tests/03_wakeup_sources/003_no_method_available.sh new file mode 100644 index 00000000000..886d89d1cb6 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/003_no_method_available.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# check that a watcher whose methods are all unavailable (Windows ones, here) +# and that doesn't accept polling isn't registered + +set -ex + +. "$(dirname "$0")/../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +case "$(uname -s)" in + *MINGW*|*MSYS*|*CYGWIN*) exit 0 ;; +esac + +touch "$TESTFILE" + +start_reactor "$POLICY_FILE" +wait_for_log "Reactor events promise '$PROMISE_NAME': none of its wakeup sources is available on this host" +wait_for_watchers 0 diff --git a/tests/reactor/tests/03_wakeup_sources/004_poll_only.cf b/tests/reactor/tests/03_wakeup_sources/004_poll_only.cf new file mode 100644 index 00000000000..b0277eebfcb --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/004_poll_only.cf @@ -0,0 +1,31 @@ +####################################################### +# +# Test that wakeup_sources => { "poll" } polls +# +####################################################### +body common control +{ + inputs => { "../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when deleted(file) +{ + file_deleted => "$(file)"; + interval => 1; + wakeup_sources => { "poll" }; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + + events: + "$(promise_name)" + when => deleted("$(test)"), + then => record_event("$(events_file)", "$(test) was deleted"); +} diff --git a/tests/reactor/tests/03_wakeup_sources/004_poll_only.sh b/tests/reactor/tests/03_wakeup_sources/004_poll_only.sh new file mode 100644 index 00000000000..d32f63f6864 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/004_poll_only.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# check that a watcher limited to polling uses it, and still fires + +set -ex + +. "$(dirname "$0")/../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +touch "$TESTFILE" + +start_reactor "$POLICY_FILE" +wait_for_watchers 1 +wait_for_log "File '$TESTFILE' is present, watching for its deletion" +wait_until log_contains "Reactor watcher '$PROMISE_NAME' uses wakeup source 'poll'" + +rm "$TESTFILE" +wait_for_lines "$EVENTS_FILE" 1 +check_event "$EVENTS_FILE" 1 "$TESTFILE was deleted" diff --git a/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.cf b/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.cf new file mode 100644 index 00000000000..d9be876cb84 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.cf @@ -0,0 +1,39 @@ +####################################################### +# +# Test that changing wakeup_sources doesn't reset the state of a watcher +# +####################################################### +body common control +{ + inputs => { "../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when deleted(file, methods) +{ + file_deleted => "$(file)"; + wakeup_sources => { @(methods) }; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + "methods_file" string => getenv("METHODS_FILE", 999); + "reload_marker" string => getenv("RELOAD_MARKER", 999); + "methods" slist => readstringlist("$(methods_file)", "", "\n", 10, 1000); + + # Evaluated on start and on every policy reload, after the watchers were + # set aside: tells the test it can change the file, and keeps the watchers + # paused long enough for it to do so + "slow" + string => execresult("touch '$(reload_marker)' && sleep 3", "useshell"); + + events: + "$(promise_name)" + when => deleted("$(test)", @(methods)), + then => record_event("$(events_file)", "$(test) was deleted"); +} diff --git a/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.sh b/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.sh new file mode 100644 index 00000000000..952a67b6dd8 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/005_methods_change_keeps_state.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# check that a watcher whose 'wakeup_sources' changed over a policy reload still +# takes over the state of the watcher from before the reload, so that a +# deletion while reloading fires. With the methods part of the watcher's +# identity, it would start from scratch and only see the file missing. + +set -ex + +. "$(dirname "$0")/../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +METHODS_FILE="$WORKDIR/tmp/methods" +RELOAD_MARKER="$WORKDIR/tmp/reload_marker" +export METHODS_FILE RELOAD_MARKER + +echo poll > "$METHODS_FILE" +touch "$TESTFILE" + +start_reactor "$POLICY_FILE" +wait_for_watchers 1 +wait_for_log "File '$TESTFILE' is present, watching for its deletion" +wait_until log_contains "Reactor watcher '$PROMISE_NAME' uses wakeup source 'poll'" + +# a different list, that still works everywhere +printf 'inotify\nrdcw\npoll\n' > "$METHODS_FILE" +rm "$RELOAD_MARKER" +kill -HUP "$reactor_pid" +wait_for_log "Rereading policy file" +if ! wait_until [ -f "$RELOAD_MARKER" ]; then + echo "Timed out waiting for the policy to be reloaded, reactor log:" >&2 + cat "$LOG" >&2 + exit 1 +fi +rm "$TESTFILE" + +wait_for_lines "$EVENTS_FILE" 1 +check_event "$EVENTS_FILE" 1 "$TESTFILE was deleted" diff --git a/tests/reactor/tests/03_wakeup_sources/006_repair_converges.cf b/tests/reactor/tests/03_wakeup_sources/006_repair_converges.cf new file mode 100644 index 00000000000..f472a606fb9 --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/006_repair_converges.cf @@ -0,0 +1,39 @@ +####################################################### +# +# Test that a bundle repairing the watched file converges +# +####################################################### +body common control +{ + inputs => { "../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when content_modified(file) +{ + file_content_modified => "$(file)"; + interval => 1; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + + events: + "$(promise_name)" + when => content_modified("$(test)"), + then => { + record_event("$(events_file)", "$(test) was modified"), + repair("$(test)"), + }; +} + +bundle agent repair(file) +{ + files: + "$(file)" content => "good"; +} diff --git a/tests/reactor/tests/03_wakeup_sources/006_repair_converges.sh b/tests/reactor/tests/03_wakeup_sources/006_repair_converges.sh new file mode 100644 index 00000000000..c8dc12c9aaa --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/006_repair_converges.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# check that a 'then' bundle repairing the watched file runs once more, for +# its own change, and that this converges: the second run changes nothing, so +# there is no third. Changes made while the bundles run are never lost. + +set -ex + +. "$(dirname "$0")/../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +echo good > "$TESTFILE" + +start_reactor "$POLICY_FILE" +wait_for_watchers 1 + +echo bad > "$TESTFILE" +wait_for_lines "$EVENTS_FILE" 1 +wait_until grep -q -x good "$TESTFILE" + +# the run for its own repair +wait_for_lines "$EVENTS_FILE" 2 + +# and no more, give it a few checks +sleep 3 +wait_for_lines "$EVENTS_FILE" 2 + +# and the watcher still fires on a later change +echo bad again > "$TESTFILE" +wait_for_lines "$EVENTS_FILE" 3 diff --git a/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.cf b/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.cf new file mode 100644 index 00000000000..3f0b44e60de --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.cf @@ -0,0 +1,30 @@ +####################################################### +# +# Test the limit on how often a watcher runs its bundles +# +####################################################### +body common control +{ + inputs => { "../../lib.cf" }; + bundlesequence => { "test" }; + version => "1.0"; +} + +body when modified(file) +{ + file_modified => "$(file)"; + interval => 1; +} + +bundle reactor test +{ + vars: + "test" string => getenv("TESTFILE", 999); + "events_file" string => getenv("EVENTS_FILE", 999); + "promise_name" string => basename("$(this.promise_filename)", ".cf"); + + events: + "$(promise_name)" + when => modified("$(test)"), + then => record_event("$(events_file)", "$(test) was modified"); +} diff --git a/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.sh b/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.sh new file mode 100644 index 00000000000..d2f1c7441cf --- /dev/null +++ b/tests/reactor/tests/03_wakeup_sources/007_trigger_limit.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# check that a watcher firing more than 10 times a minute is delayed, its +# further events coalesced into one run once the delay is over + +set -ex + +. "$(dirname "$0")/../../lib.sh" + +POLICY_FILE="$1" + +TESTFILE="$WORKDIR/tmp/test" +EVENTS_FILE="$WORKDIR/tmp/events" +export TESTFILE EVENTS_FILE + +touch "$TESTFILE" + +start_reactor "$POLICY_FILE" +wait_for_watchers 1 + +# Each modification seen as its own event: wait for each run before the next +i=0 +while [ $i -lt 10 ]; do + i=$((i + 1)) + touch "$TESTFILE" + wait_for_lines "$EVENTS_FILE" $i +done + +# Beyond the limit, a burst of modifications only runs the bundles once more, +# after the delay +i=0 +while [ $i -lt 5 ]; do + i=$((i + 1)) + touch "$TESTFILE" + sleep 1.1 2>/dev/null || sleep 2 +done +wait_for_log "fired more than 10 times in 60 seconds, delaying its next run by 10 seconds" +wait_for_lines "$EVENTS_FILE" 11 +sleep 3 +wait_for_lines "$EVENTS_FILE" 11 diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index d5084bafb22..338f4fc9c63 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -235,12 +235,15 @@ regex_test_SOURCES = regex_test.c ../../libpromises/match_scope.c watcher_test_SOURCES = watcher_test.c \ ../../cf-reactor/watcher.c \ + ../../cf-reactor/watcher_report.c \ + ../../cf-reactor/watch_backends.c \ + ../../cf-reactor/watch_location.c \ ../../cf-reactor/file_watcher.c \ ../../cf-reactor/stoppable_thread.c \ ../../cf-reactor/wakeup_channel.c -file_watcher_test_SOURCES = file_watcher_test.c \ - ../../cf-reactor/file_watcher.c +# The test includes file_watcher.c, to reach its static helpers +file_watcher_test_SOURCES = file_watcher_test.c user_watcher_test_SOURCES = user_watcher_test.c diff --git a/tests/unit/file_watcher_test.c b/tests/unit/file_watcher_test.c index 9ff463f88bb..4ba1d6056c4 100644 --- a/tests/unit/file_watcher_test.c +++ b/tests/unit/file_watcher_test.c @@ -1,6 +1,6 @@ #include -#include /* FileWatcherStateNew(), FileWatcherCheckFileDeleted() */ +#include /* Included for HashFileContent() */ #include /* RlistAppendScalar() */ #include /* LoggingPrivContext, LoggingPrivSetContext() */ @@ -203,6 +203,39 @@ static void test_check_file_deleted_logs_error_once(void) TearDownSymlinkLoop(dir, loop); } +/** A file swapped for a FIFO after being lstat()ed must fail to be hashed, + * not block on opening it for reading */ +static void test_hash_file_content_fifo_fails_fast(void) +{ + char dir[] = "/tmp/file_watcher_test.XXXXXX"; + assert_true(mkdtemp(dir) != NULL); + char path[PATH_MAX]; + xsnprintf(path, sizeof(path), "%s/fifo", dir); + assert_int_equal(mkfifo(path, 0600), 0); + + /* Would hang the test if it blocked, so the alarm turns that into a failure */ + alarm(10); + unsigned char digest[EVP_MAX_MD_SIZE + 1]; + assert_false(HashFileContent(path, digest)); + alarm(0); + + unlink(path); + assert_int_equal(rmdir(dir), 0); +} + +static void test_hash_file_content_regular(void) +{ + char dir[] = "/tmp/file_watcher_test.XXXXXX"; + char path[PATH_MAX]; + CreateTempFile(dir, path, sizeof(path)); + + unsigned char digest[EVP_MAX_MD_SIZE + 1]; + assert_true(HashFileContent(path, digest)); + + unlink(path); + assert_int_equal(rmdir(dir), 0); +} + int main() { PRINT_TEST_BANNER(); @@ -213,6 +246,8 @@ int main() unit_test(test_check_file_deleted_dangling_symlink), unit_test(test_check_file_deleted_error), unit_test(test_check_file_deleted_logs_error_once), + unit_test(test_hash_file_content_fifo_fails_fast), + unit_test(test_hash_file_content_regular), }; return run_tests(tests); diff --git a/tests/unit/watcher_test.c b/tests/unit/watcher_test.c index b74308b8901..8b74e05f55b 100644 --- a/tests/unit/watcher_test.c +++ b/tests/unit/watcher_test.c @@ -5,24 +5,65 @@ #include /* Promise */ #include /* RlistAppendScalar() */ #include /* StringEqual() */ +#include /* FileRead() */ +#include /* REACTOR_WATCHERS_REPORT_FILE */ +#include /* MountinfoPathIsRemote(), WatchLocationSetMountTable() */ #include /* snprintf() */ #include /* mkdtemp() */ #include /* close(), unlink(), rmdir(), symlink(), sleep() */ #include /* open() */ +#include /* mkdir() */ +#include +#include /* alarm() */ /* The registry only hashes the expanded promise it is given (its promiser - * here, as it has no constraints, comment or parent) and hands a copy of it - * back to the WatcherEventFn, so minimal promises are enough. Their org_pp - * tells apart the identical promises of different policies. */ + * and bundle here, as it has no constraints or comment) and hands a copy of + * it back to the WatcherEventFn, so minimal promises are enough. Their org_pp + * tells apart the identical promises of different policies. The bundle is + * there for the watchers report. */ +static Bundle test_bundle = { .name = "test", .ns = "default" }; +static BundleSection test_section = { .parent_bundle = &test_bundle }; static const Promise unexpanded_promise; static const Promise expanded_promise = { .promiser = "test-event", .org_pp = &unexpanded_promise, + .parent_section = &test_section, }; -/* Returns the options of a file_deleted watcher, valid until the next call - * (WatcherRegister() copies them) */ +/* Checks of file_deleted watchers that told whether the file was deleted + * (not WATCHER_CHECK_ERROR), to wait for a reference to be taken */ +static int file_checks_ok = 0; + +static WatcherCheckResult CountedCheckFileDeleted(void *state) +{ + const WatcherCheckResult check = FileWatcherCheckFileDeleted(state); + if (check != WATCHER_CHECK_ERROR) + { + __atomic_add_fetch(&file_checks_ok, 1, __ATOMIC_SEQ_CST); + } + return check; +} + +static int FileChecksOk(void) +{ + return __atomic_load_n(&file_checks_ok, __ATOMIC_SEQ_CST); +} + +/** Waits up to 5 seconds for `count` successful file checks in total */ +static void WaitForFileChecksOk(int count) +{ + for (int i = 0; i < 50 && FileChecksOk() < count; i++) + { + usleep(100000); + } + assert_true(FileChecksOk() >= count); +} + +/* Returns the options of a file_deleted watcher of `path`, which is also its + * target, so the mount table decides its lane (see FakeRemoteMount()). Valid + * until the next call (WatcherRegister() copies them), `path` must outlive + * WatcherRegister(). */ static WatcherOptions *FileDeletedOptions(const char *path, time_t interval) { static WatcherOptions opt; @@ -31,14 +72,46 @@ static WatcherOptions *FileDeletedOptions(const char *path, time_t interval) RlistAppendScalar(&args, path); opt = (WatcherOptions) { .state = FileWatcherStateNew(NULL, args), - .check_callback = FileWatcherCheckFileDeleted, + .check_callback = CountedCheckFileDeleted, .destroy_state = FileWatcherStateDestroy, .interval = interval, + .target = { .type = REACTOR_WATCH_TARGET_FILE, .path = path }, }; RlistDestroy(args); return &opt; } +/** Makes `mount_point` an NFS mount for WatchTargetIsRemote(), until + * RemoveFakeRemoteMount() */ +static void FakeRemoteMount(const char *mount_point) +{ + char mountinfo[PATH_MAX + 256]; + xsnprintf(mountinfo, sizeof(mountinfo), + "22 1 8:1 / / rw - ext4 /dev/sda1 rw\n" + "40 22 0:50 / %s rw - nfs4 srv:/export rw\n", mount_point); + WatchLocationSetMountTable(mountinfo); +} + +/** Back to the empty mount table of the tests, where everything is local */ +static void RemoveFakeRemoteMount(void) +{ + WatchLocationSetMountTable(""); +} + +/** Polls `condition` every 0.1 s up to tenths/10 s, returns whether it held */ +static bool WaitUntil(bool (*condition)(void), int tenths) +{ + for (int i = 0; i < tenths; i++) + { + if (condition()) + { + return true; + } + usleep(100000); + } + return condition(); +} + /* Creates the file 'file' in a new temporary directory dir */ static void CreateTempFile(char *dir, char *path, size_t path_size) { @@ -161,8 +234,10 @@ static void test_watcher_register_iterations(void) /* The iterations of a promise (same promise, other promisers) are * different watchers */ - const Promise iteration_a = { .promiser = "/a", .org_pp = &unexpanded_promise }; - const Promise iteration_b = { .promiser = "/b", .org_pp = &unexpanded_promise }; + const Promise iteration_a = { .promiser = "/a", .org_pp = &unexpanded_promise, + .parent_section = &test_section }; + const Promise iteration_b = { .promiser = "/b", .org_pp = &unexpanded_promise, + .parent_section = &test_section }; WatcherRegister(&iteration_a, FileDeletedOptions("/nonexistent/a", 5)); WatcherRegister(&iteration_b, FileDeletedOptions("/nonexistent/b", 5)); @@ -185,6 +260,7 @@ static void test_watcher_register_identical_ignored(void) const Promise other_expanded_promise = { .promiser = "test-event", .org_pp = &other_unexpanded_promise, + .parent_section = &test_section, }; assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 1))); assert_true(WatcherRegister(&other_expanded_promise, FileDeletedOptions(path, 1))); @@ -373,6 +449,7 @@ static void test_event_watcher_state_kept_across_pause(void) const Promise new_expanded_promise = { .promiser = "test-event", .org_pp = &new_unexpanded_promise, + .parent_section = &test_section, }; WatcherRegister(&new_expanded_promise, FileDeletedOptions(path, 1)); EventWatcherResume(); @@ -412,10 +489,12 @@ static void test_event_watcher_state_kept_identical_ignored(void) const Promise new_expanded_promise = { .promiser = "test-event", .org_pp = &new_unexpanded_promise, + .parent_section = &test_section, }; const Promise other_expanded_promise = { .promiser = "test-event", .org_pp = &other_unexpanded_promise, + .parent_section = &test_section, }; WatcherRegister(&new_expanded_promise, FileDeletedOptions(path, 1)); WatcherRegister(&other_expanded_promise, FileDeletedOptions(path, 1)); @@ -452,6 +531,7 @@ static void test_event_watcher_state_not_kept_for_other_promise(void) const Promise other_expanded_promise = { .promiser = "other-event", .org_pp = &unexpanded_promise, + .parent_section = &test_section, }; WatcherRegister(&other_expanded_promise, FileDeletedOptions(path, 1)); EventWatcherResume(); @@ -474,8 +554,10 @@ static void test_event_watcher_several_events(void) CreateTempFile(dir_b, path_b, sizeof(path_b)); WatcherRegistryInitialize(); - const Promise iteration_a = { .promiser = "a", .org_pp = &unexpanded_promise }; - const Promise iteration_b = { .promiser = "b", .org_pp = &unexpanded_promise }; + const Promise iteration_a = { .promiser = "a", .org_pp = &unexpanded_promise, + .parent_section = &test_section }; + const Promise iteration_b = { .promiser = "b", .org_pp = &unexpanded_promise, + .parent_section = &test_section }; WatcherRegister(&iteration_a, FileDeletedOptions(path_a, 1)); WatcherRegister(&iteration_b, FileDeletedOptions(path_b, 1)); @@ -551,8 +633,9 @@ static void test_event_watcher_check_failure_reported_once(void) assert_false(WaitForQueuedEvent(fd, 30)); /* Once a check succeeds again, the next failure is reported again */ + const int checks_ok = FileChecksOk(); assert_int_equal(unlink(loop), 0); - sleep(2); + WaitForFileChecksOk(checks_ok + 1); assert_int_equal(symlink(loop, loop), 0); assert_true(WaitForQueuedEvent(fd, 50)); HandleEvents(fd); @@ -575,15 +658,20 @@ static void test_watcher_register_check_failure(void) const LogLevel old_level = LogGetGlobalLevel(); LogSetGlobalLevel(LOG_LEVEL_CRIT); - /* Without a reference, the watcher is not registered (its state is - * destroyed by the registry, checked by valgrind/ASan) */ + /* Without a reference, the watcher is still registered: its first + * successful check becomes the reference */ WatcherRegistryInitialize(); - assert_false(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 1))); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 1))); - /* So the watcher thread doesn't check it, and doesn't report its failure */ + /* Its failing checks are reported once, as a failure, not as an event */ int fd = -1; assert_true(EventWatcherInitialize(&fd)); - assert_false(WaitForQueuedEvent(fd, 20)); + assert_true(WaitForQueuedEvent(fd, 50)); + ResetRecordedEvents(); + HandleEvents(fd); + assert_int_equal(handled_event_count, 1); + assert_int_equal(last_event_check, WATCHER_CHECK_ERROR); + assert_false(WaitForQueuedEvent(fd, 30)); EventWatcherFinalize(); @@ -591,9 +679,1004 @@ static void test_watcher_register_check_failure(void) TearDownSymlinkLoop(dir, loop); } +static void test_watcher_check_failure_retried_before_interval(void) +{ + char dir[] = "/tmp/watcher_test.XXXXXX"; + char loop[PATH_MAX]; + char path[PATH_MAX]; + SetUpSymlinkLoop(dir, loop, sizeof(loop), path, sizeof(path)); + + const LogLevel old_level = LogGetGlobalLevel(); + LogSetGlobalLevel(LOG_LEVEL_CRIT); + + /* Registered while its check fails, with an interval longer than the + * test */ + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 300))); + + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + assert_true(WaitForQueuedEvent(fd, 50)); + ResetRecordedEvents(); + HandleEvents(fd); + assert_int_equal(last_event_check, WATCHER_CHECK_ERROR); + + /* Once the file can be checked, the reference is taken well before the + * interval, so a deletion checked for on a wakeup fires */ + assert_int_equal(unlink(loop), 0); + assert_int_equal(mkdir(loop, 0700), 0); + int file_fd = open(path, O_CREAT | O_WRONLY, 0600); + assert_true(file_fd >= 0); + close(file_fd); + WaitForFileChecksOk(FileChecksOk() + 1); + assert_int_equal(unlink(path), 0); + WatcherMarkAllDirty(); + assert_true(WaitForQueuedEvent(fd, 50)); + HandleEvents(fd); + assert_int_equal(handled_event_count, 2); + assert_int_equal(last_event_check, WATCHER_CHECK_EVENT); + + EventWatcherFinalize(); + + LogSetGlobalLevel(old_level); + assert_int_equal(rmdir(loop), 0); + assert_int_equal(rmdir(dir), 0); +} + +/** Fires from the interval alone, which is shorter than the longest sleep of + * the watcher thread, so it sleeps until the watcher is due, not just up to + * that longest sleep */ +static void test_event_watcher_fires_at_interval(void) +{ + char dir[] = "/tmp/watcher_test.XXXXXX"; + char path[PATH_MAX]; + CreateTempFile(dir, path, sizeof(path)); + + const int checks_ok = FileChecksOk(); + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + + /* The check of the registration takes the reference, then the first + * pass of the watcher thread checks again */ + WaitForFileChecksOk(checks_ok + 2); + ResetRecordedEvents(); + assert_int_equal(unlink(path), 0); + + assert_true(WaitForQueuedEvent(fd, 50)); + HandleEvents(fd); + assert_int_equal(handled_event_count, 1); + + EventWatcherFinalize(); + assert_int_equal(rmdir(dir), 0); +} + +static bool ReportMtime(const char *report_path, struct timespec *mtime) +{ + struct stat sb; + if (stat(report_path, &sb) != 0) + { + return false; + } +#if defined(HAVE_STRUCT_STAT_ST_MTIM) + *mtime = sb.st_mtim; +#elif defined(HAVE_STRUCT_STAT_ST_MTIMESPEC) + *mtime = sb.st_mtimespec; +#else + *mtime = (struct timespec) { .tv_sec = sb.st_mtime, .tv_nsec = 0 }; +#endif + return true; +} + +/** The report is only rewritten when it changes, and removed when cf-reactor + * finishes */ +static void test_watchers_report_written_when_changed_removed_on_finalize(void) +{ + char workdir[] = "/tmp/watcher_test.XXXXXX"; + assert_true(mkdtemp(workdir) != NULL); + char state_dir[64]; + xsnprintf(state_dir, sizeof(state_dir), "%s/state", workdir); + assert_int_equal(mkdir(state_dir, 0700), 0); + char report_path[128]; + xsnprintf(report_path, sizeof(report_path), "%s/%s", state_dir, REACTOR_WATCHERS_REPORT_FILE); + assert_int_equal(setenv("CFENGINE_TEST_OVERRIDE_WORKDIR", workdir, 1), 0); + + WatcherRegistryInitialize(); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions("/nonexistent/test-event", 300))); + EventWatcherWakeUp(); + struct timespec first; + assert_true(ReportMtime(report_path, &first)); + + /* Same watchers, as after a policy re-read that changed nothing */ + usleep(50000); + EventWatcherWakeUp(); + struct timespec second; + assert_true(ReportMtime(report_path, &second)); + assert_true(first.tv_sec == second.tv_sec && first.tv_nsec == second.tv_nsec); + + /* A watcher of a bundle of another namespace, which the report tells apart */ + static Bundle other_bundle = { .name = "test", .ns = "other" }; + static BundleSection other_section = { .parent_bundle = &other_bundle }; + static const Promise other_unexpanded_promise; + const Promise other_promise = { + .promiser = "other-event", + .org_pp = &other_unexpanded_promise, + .parent_section = &other_section, + }; + assert_true(WatcherRegister(&other_promise, FileDeletedOptions("/nonexistent/other-event", 300))); + usleep(50000); + EventWatcherWakeUp(); + struct timespec third; + assert_true(ReportMtime(report_path, &third)); + assert_false(first.tv_sec == third.tv_sec && first.tv_nsec == third.tv_nsec); + + /* Sorted by bundle, whatever the order of registration */ + Writer *content = FileRead(report_path, SIZE_MAX, NULL); + assert_true(content != NULL); + const char *test_bundle_entry = strstr(StringWriterData(content), "\"bundle\":\"test\""); + if (test_bundle_entry == NULL) + { + test_bundle_entry = strstr(StringWriterData(content), "\"bundle\": \"test\""); + } + const char *other_bundle_entry = strstr(StringWriterData(content), "other:test"); + assert_true(test_bundle_entry != NULL); + assert_true(other_bundle_entry != NULL); + assert_true(other_bundle_entry < test_bundle_entry); + WriterClose(content); + + EventWatcherFinalize(); + assert_int_not_equal(access(report_path, F_OK), 0); + + unsetenv("CFENGINE_TEST_OVERRIDE_WORKDIR"); + assert_int_equal(rmdir(state_dir), 0); + assert_int_equal(rmdir(workdir), 0); +} + +/* A watcher whose second check (the first one of the watcher thread, the + * first being WatcherRegister()'s) blocks until ReleaseBlockedCheck(), as on + * an unresponsive NFS mount, then reports an event. Later checks find + * nothing. One such check at a time. */ +typedef struct +{ + int checks; + int block_on; /* the check that blocks */ + bool in_check; +} BlockingState; + +static pthread_mutex_t blocking_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t blocking_cond = PTHREAD_COND_INITIALIZER; +static bool blocked_check_entered = false; +static bool blocked_check_released = false; +static int blocking_states_destroyed = 0; + +static WatcherCheckResult BlockingCheck(void *state) +{ + BlockingState *bs = state; + bs->checks++; + if (bs->checks != bs->block_on) + { + return WATCHER_CHECK_NO_EVENT; + } + + /* Like a hard NFS mount, not cancellable, see StoppableThreadStop() */ + int old_cancel_state; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_cancel_state); + pthread_mutex_lock(&blocking_mutex); + bs->in_check = true; + blocked_check_entered = true; + pthread_cond_broadcast(&blocking_cond); + while (!blocked_check_released) + { + pthread_cond_wait(&blocking_cond, &blocking_mutex); + } + bs->in_check = false; + pthread_mutex_unlock(&blocking_mutex); + pthread_setcancelstate(old_cancel_state, NULL); + return WATCHER_CHECK_EVENT; +} + +static void BlockingStateDestroy(void *state) +{ + BlockingState *bs = state; + /* Never while its check runs */ + assert_false(bs->in_check); + pthread_mutex_lock(&blocking_mutex); + blocking_states_destroyed++; + pthread_mutex_unlock(&blocking_mutex); + free(bs); +} + +/** Options of a blocking watcher whose check number `block_on` blocks, of the + * file at `path` (only for the lane it gets, see WatchTargetIsRemote()), or + * NULL. Its check number 1 is WatcherRegister()'s, unless remote. */ +static WatcherOptions *BlockingOptions(const char *path, int block_on) +{ + static WatcherOptions opt; + BlockingState *bs = xcalloc(1, sizeof(BlockingState)); + bs->block_on = block_on; + opt = (WatcherOptions) { + .state = bs, + .check_callback = BlockingCheck, + .destroy_state = BlockingStateDestroy, + .interval = 300, + .target = { .type = REACTOR_WATCH_TARGET_FILE, .path = path }, + }; + return &opt; +} + +static void ResetBlockedCheck(void) +{ + pthread_mutex_lock(&blocking_mutex); + blocked_check_entered = false; + blocked_check_released = false; + blocking_states_destroyed = 0; + pthread_mutex_unlock(&blocking_mutex); +} + +static void WaitForBlockedCheck(void) +{ + pthread_mutex_lock(&blocking_mutex); + while (!blocked_check_entered) + { + pthread_cond_wait(&blocking_cond, &blocking_mutex); + } + pthread_mutex_unlock(&blocking_mutex); +} + +static void ReleaseBlockedCheck(void) +{ + pthread_mutex_lock(&blocking_mutex); + blocked_check_released = true; + pthread_cond_broadcast(&blocking_cond); + pthread_mutex_unlock(&blocking_mutex); +} + +static int BlockingStatesDestroyed(void) +{ + pthread_mutex_lock(&blocking_mutex); + const int destroyed = blocking_states_destroyed; + pthread_mutex_unlock(&blocking_mutex); + return destroyed; +} + +static bool OneBlockingStateDestroyed(void) +{ + return (BlockingStatesDestroyed() >= 1); +} + +static void *ReleaseBlockedCheckLater(ARG_UNUSED void *arg) +{ + usleep(500000); + ReleaseBlockedCheck(); + return NULL; +} + +static time_t ElapsedSecs(const struct timespec *start) +{ + struct timespec end; + clock_gettime(CLOCK_MONOTONIC, &end); + return end.tv_sec - start->tv_sec; +} + +static const Promise blocking_promise = { + .promiser = "blocking-event", + .org_pp = &unexpanded_promise, + .parent_section = &test_section, +}; + +/** A blocked check doesn't block the main thread registering other watchers or + * waking the watcher thread up, and its event fires once it returns */ +static void test_blocked_check_does_not_block_registry(void) +{ + alarm(60); /* a deadlock fails the test instead of hanging it */ + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(NULL, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions("/nonexistent/test-event", 300))); + EventWatcherWakeUp(); + assert_true(ElapsedSecs(&start) < 2); + + assert_false(WaitForQueuedEvent(fd, 5)); + ReleaseBlockedCheck(); + assert_true(WaitForQueuedEvent(fd, 50)); + HandleEvents(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&blocking_promise); + assert_int_equal(last_event_check, WATCHER_CHECK_EVENT); + + EventWatcherFinalize(); + assert_int_equal(BlockingStatesDestroyed(), 1); + alarm(0); +} + +/** Pausing, re-reading a policy without the watcher, and resuming don't wait + * for its blocked check. The event it then observes is dropped with the + * watcher, which is only freed once its check is over. */ +static void test_blocked_check_watcher_removed(void) +{ + alarm(60); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(NULL, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + EventWatcherPause(NULL, RecordEvent); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions("/nonexistent/test-event", 300))); + EventWatcherResume(); + assert_true(ElapsedSecs(&start) < 2); + assert_int_equal(BlockingStatesDestroyed(), 0); + + ReleaseBlockedCheck(); + assert_true(WaitUntil(OneBlockingStateDestroyed, 50)); + assert_int_equal(BlockingStatesDestroyed(), 1); + assert_false(WaitForQueuedEvent(fd, 10)); + assert_int_equal(handled_event_count, 0); + + EventWatcherFinalize(); + alarm(0); +} + +static void HandleEventsSince(int fd) +{ + assert_true(WaitForQueuedEvent(fd, 50)); + HandleEvents(fd); +} + +/** Handles events for up to tenths/10 s, until `count` were handled in total. + * A wakeup may be left from events already handled (a lane pushes an event, + * then notifies, so a pop in between leaves the wakeup), so a wakeup doesn't + * mean an event. */ +static void HandleEventsUntil(int fd, int count, int tenths) +{ + for (int i = 0; i < tenths && handled_event_count < count; i++) + { + if (WaitForQueuedEvent(fd, 1)) + { + HandleEvents(fd); + } + } + assert_int_equal(handled_event_count, count); +} + +/** Asserts that no event is handled within tenths/10 s, see + * HandleEventsUntil() */ +static void AssertNoEventHandled(int fd, int tenths) +{ + const int count = handled_event_count; + for (int i = 0; i < tenths; i++) + { + if (WaitForQueuedEvent(fd, 1)) + { + HandleEvents(fd); + } + } + assert_int_equal(handled_event_count, count); +} + +/** Re-reads the policy with an identical blocking watcher, i.e. a new + * unexpanded promise but the same expanded one, and returns the latter */ +static void ReReadWithIdenticalBlockingWatcher(const Promise *unexpanded, Promise *expanded) +{ + *expanded = (Promise) { + .promiser = "blocking-event", + .org_pp = unexpanded, + .parent_section = &test_section, + }; + EventWatcherPause(NULL, RecordEvent); + assert_true(WatcherRegister(expanded, BlockingOptions(NULL, 2))); + EventWatcherResume(); +} + +/** The identical watcher of the re-read policy is registered without waiting + * for the blocked check: it shares the state, and so the event that check + * observed. Its own state is destroyed right away. */ +static void test_blocked_check_state_taken_over(void) +{ + alarm(60); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(NULL, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + static const Promise new_unexpanded_promise; + Promise new_blocking_promise; + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + ReReadWithIdenticalBlockingWatcher(&new_unexpanded_promise, &new_blocking_promise); + assert_true(ElapsedSecs(&start) < 2); + + assert_false(WaitForQueuedEvent(fd, 5)); + assert_int_equal(BlockingStatesDestroyed(), 1); + + ReleaseBlockedCheck(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&new_blocking_promise); + assert_int_equal(last_event_check, WATCHER_CHECK_EVENT); + assert_int_equal(BlockingStatesDestroyed(), 1); + + /* Fires once */ + assert_false(WaitForQueuedEvent(fd, 15)); + + EventWatcherFinalize(); + assert_int_equal(BlockingStatesDestroyed(), 2); + alarm(0); +} + +/** Re-read twice while the check is blocked: the watcher of the last policy + * gets the event, the one in between is dropped, and their own states are + * destroyed right away */ +static void test_blocked_check_state_taken_over_after_two_rereads(void) +{ + alarm(60); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(NULL, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + static const Promise first_unexpanded_promise; + static const Promise second_unexpanded_promise; + Promise first_promise; + Promise second_promise; + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + ReReadWithIdenticalBlockingWatcher(&first_unexpanded_promise, &first_promise); + ReReadWithIdenticalBlockingWatcher(&second_unexpanded_promise, &second_promise); + assert_true(ElapsedSecs(&start) < 2); + assert_int_equal(BlockingStatesDestroyed(), 2); + + ReleaseBlockedCheck(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&second_promise); + assert_int_equal(last_event_check, WATCHER_CHECK_EVENT); + assert_int_equal(BlockingStatesDestroyed(), 2); + assert_false(WaitForQueuedEvent(fd, 15)); + + EventWatcherFinalize(); + assert_int_equal(BlockingStatesDestroyed(), 3); + alarm(0); +} + +/** Finalized while paused, before the blocked check is over: every state is + * destroyed once (twice would fail BlockingStateDestroy() or valgrind), the + * shared one by the last of the watcher thread and the two watchers to drop + * it */ +static void test_blocked_check_finalized_before_handoff(void) +{ + alarm(60); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(NULL, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + static const Promise new_unexpanded_promise; + const Promise new_blocking_promise = { + .promiser = "blocking-event", + .org_pp = &new_unexpanded_promise, + .parent_section = &test_section, + }; + EventWatcherPause(NULL, RecordEvent); + assert_true(WatcherRegister(&new_blocking_promise, BlockingOptions(NULL, 2))); + + /* Finalizing joins the watcher thread, so release the check meanwhile */ + pthread_t releaser; + assert_int_equal(pthread_create(&releaser, NULL, ReleaseBlockedCheckLater, NULL), 0); + EventWatcherFinalize(); + assert_int_equal(pthread_join(releaser, NULL), 0); + assert_int_equal(BlockingStatesDestroyed(), 2); + assert_int_equal(handled_event_count, 0); + alarm(0); +} + +/* A mount table in the format of /proc/self/mountinfo, with the escaped + * mount point "/mnt/my share", stacked mounts and fuse */ +static const char *const MOUNTINFO = + "22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw,errors=remount-ro\n" + "40 22 0:50 / /mnt rw,relatime shared:30 - nfs4 srv:/export rw,vers=4.2\n" + "41 40 8:2 / /mnt/local rw,relatime shared:31 - ext4 /dev/sda2 rw\n" + "42 22 0:51 / /mnt/my\\040share rw,relatime shared:32 - cifs //srv/share rw\n" + "43 22 0:52 / /home/user/sshfs rw,nosuid,nodev,relatime shared:33 - fuse.sshfs u@h:/ rw\n" + "44 22 0:53 / /data rw,relatime - nfs srv:/data rw\n" + "45 22 8:3 / /data rw,relatime - xfs /dev/sda3 rw\n" + "46 22 0:54 /sub /srv/nfs rw,relatime master:7 shared:9 - nfs srv:/sub rw\n" + "garbage line\n" + "47 22 0:55 / /broken rw,relatime\n"; + +static void test_mountinfo_path_classification(void) +{ + /* The longest mount point containing the path decides */ + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/etc/passwd")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/file")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/local/file")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/local")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/localfile")); + + /* Whole components only */ + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mntx/file")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/srv/nfs2/file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/srv/nfs/file")); + + /* Escaped mount point */ + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/my share/file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/my/file")); /* under /mnt */ + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/home/user/file")); + + /* fuse counts as remote */ + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/home/user/sshfs/file")); + + /* The last of stacked mounts hides the others */ + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/data/file")); + + /* Repeated and trailing slashes don't hide a mount point */ + assert_true(MountinfoPathIsRemote(MOUNTINFO, "//mnt//file")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/local/")); + + /* Nor do "." and "..", collapsed lexically */ + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/local/../file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/./file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/etc/../mnt/file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/../../mnt")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/..")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/../etc/file")); + assert_false(MountinfoPathIsRemote(MOUNTINFO, "/mnt/./local/./file")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/.local")); + assert_true(MountinfoPathIsRemote(MOUNTINFO, "/mnt/..local")); + + /* Relative paths and empty tables are local */ + assert_false(MountinfoPathIsRemote(MOUNTINFO, "mnt/file")); + assert_false(MountinfoPathIsRemote("", "/mnt/file")); +} + +static void test_mountinfo_fs_types(void) +{ + const char *const remote[] = { "nfs", "nfs4", "cifs", "smb3", "smbfs", "9p", "fuse.sshfs", + "ceph", "glusterfs", "afs", "lustre", "fuse", "fuse.rclone", + "ocfs2", "gfs2", "virtiofs", "beegfs", "panfs", "ncpfs" }; + const char *const local[] = { "ext4", "xfs", "btrfs", "tmpfs", "overlay", "nfsd", "fusectl" }; + char mountinfo[256]; + for (size_t i = 0; i < sizeof(remote) / sizeof(remote[0]); i++) + { + xsnprintf(mountinfo, sizeof(mountinfo), "40 22 0:50 / /m rw - %s src rw\n", remote[i]); + assert_true(MountinfoPathIsRemote(mountinfo, "/m/file")); + } + for (size_t i = 0; i < sizeof(local) / sizeof(local[0]); i++) + { + xsnprintf(mountinfo, sizeof(mountinfo), "40 22 0:50 / /m rw - %s src rw\n", local[i]); + assert_false(MountinfoPathIsRemote(mountinfo, "/m/file")); + } +} + +static void test_path_is_unc(void) +{ + assert_true(WatchPathIsUNC("\\\\server\\share\\file")); + assert_true(WatchPathIsUNC("//server/share/file")); + assert_true(WatchPathIsUNC("\\\\?\\UNC\\server\\share\\file")); + assert_true(WatchPathIsUNC("\\\\?\\unc\\server\\share\\file")); + assert_false(WatchPathIsUNC("\\\\?\\C:\\file")); + assert_false(WatchPathIsUNC("\\\\.\\pipe\\name")); + assert_false(WatchPathIsUNC("C:\\file")); + assert_false(WatchPathIsUNC("\\file")); + assert_false(WatchPathIsUNC("/etc/file")); + assert_false(WatchPathIsUNC("\\\\")); +} + +static void test_watch_target_is_remote(void) +{ + /* Only files can be remote */ + const ReactorWatchTarget users = { .type = REACTOR_WATCH_TARGET_USERS }; + assert_false(WatchTargetIsRemote(&users)); + const ReactorWatchTarget no_path = { .type = REACTOR_WATCH_TARGET_FILE }; + assert_false(WatchTargetIsRemote(&no_path)); + +#ifdef __MINGW32__ + const ReactorWatchTarget unc = { .type = REACTOR_WATCH_TARGET_FILE, + .path = "\\\\server\\share\\file" }; + assert_true(WatchTargetIsRemote(&unc)); +#else + WatchLocationSetMountTable(MOUNTINFO); + const ReactorWatchTarget remote = { .type = REACTOR_WATCH_TARGET_FILE, .path = "/mnt/file" }; + assert_true(WatchTargetIsRemote(&remote)); + const ReactorWatchTarget local = { .type = REACTOR_WATCH_TARGET_FILE, .path = "/etc/file" }; + assert_false(WatchTargetIsRemote(&local)); + + /* Without a mount table, everything is local */ + WatchLocationSetMountTable(""); + assert_false(WatchTargetIsRemote(&remote)); + + /* The host's own, whatever it says, read and parsed once until forgotten */ + WatchLocationSetMountTable(NULL); + const bool remote_on_host = WatchTargetIsRemote(&local); + assert_true(WatchTargetIsRemote(&local) == remote_on_host); + WatchLocationForgetMountTable(); + assert_true(WatchTargetIsRemote(&local) == remote_on_host); + + RemoveFakeRemoteMount(); +#endif +} + +static const Promise remote_promise = { + .promiser = "remote-event", + .org_pp = &unexpanded_promise, + .parent_section = &test_section, +}; + +/** A remote watcher's first check is on its lane, not in WatcherRegister(), + * and becomes the reference */ +static void test_remote_watcher_reference_taken_on_its_lane(void) +{ + alarm(60); + char dir[] = "/tmp/watcher_test.XXXXXX"; + char path[PATH_MAX]; + CreateTempFile(dir, path, sizeof(path)); + FakeRemoteMount(dir); + ResetRecordedEvents(); + ResetBlockedCheck(); + + WatcherRegistryInitialize(); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + + /* It would block the registration if checked there */ + char blocking_path[PATH_MAX]; + xsnprintf(blocking_path, sizeof(blocking_path), "%s/blocking", dir); + const int checks_ok = FileChecksOk(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(blocking_path, 1))); + assert_true(WatcherRegister(&remote_promise, FileDeletedOptions(path, 300))); + assert_int_equal(FileChecksOk(), checks_ok); + EventWatcherWakeUp(); + WaitForBlockedCheck(); + ReleaseBlockedCheck(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&blocking_promise); + + /* The file existed at the first check, its deletion fires */ + WaitForFileChecksOk(checks_ok + 1); + assert_int_equal(unlink(path), 0); + WatcherMarkAllDirty(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 2); + AssertLastEventFor(&remote_promise); + + EventWatcherFinalize(); + RemoveFakeRemoteMount(); + assert_int_equal(rmdir(dir), 0); + alarm(0); +} + +/** A remote watcher's check blocked on an unresponsive server neither delays + * the events of the local watchers nor the registration of other remote + * ones */ +static void test_blocked_remote_check_does_not_delay_local(void) +{ + alarm(60); + char dir[] = "/tmp/watcher_test.XXXXXX"; + char path[PATH_MAX]; + CreateTempFile(dir, path, sizeof(path)); + FakeRemoteMount("/remote"); + ResetRecordedEvents(); + ResetBlockedCheck(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions("/remote/blocking", 1))); + assert_true(WatcherRegister(&expanded_promise, FileDeletedOptions(path, 1))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + assert_int_equal(unlink(path), 0); + assert_true(WaitForQueuedEvent(fd, 20)); + HandleEvents(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&expanded_promise); + + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + assert_true(WatcherRegister(&remote_promise, FileDeletedOptions("/remote/other", 300))); + EventWatcherWakeUp(); + assert_true(ElapsedSecs(&start) < 2); + + ReleaseBlockedCheck(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 2); + AssertLastEventFor(&blocking_promise); + + EventWatcherFinalize(); + assert_int_equal(BlockingStatesDestroyed(), 1); + RemoveFakeRemoteMount(); + assert_int_equal(rmdir(dir), 0); + alarm(0); +} + +/** The identical watcher of a re-read policy on the other lane (the file + * system got mounted meanwhile) gets a fresh state rather than that of the + * old watcher, which is about another file system, so it doesn't wait for a + * check of the old state stuck on it. The event that check then observes is + * dropped with it. */ +static void test_watcher_moved_to_other_lane_starts_afresh(void) +{ + alarm(60); + char dir[] = "/tmp/watcher_test.XXXXXX"; + char path[PATH_MAX]; + CreateTempFile(dir, path, sizeof(path)); + char blocking_path[PATH_MAX]; + xsnprintf(blocking_path, sizeof(blocking_path), "%s/blocking", dir); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions(blocking_path, 2))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + /* Same promise, so same hash, watching a file now on NFS */ + FakeRemoteMount(dir); + static const Promise new_unexpanded_promise; + const Promise new_blocking_promise = { + .promiser = "blocking-event", + .org_pp = &new_unexpanded_promise, + .parent_section = &test_section, + }; + const int checks_ok = FileChecksOk(); + EventWatcherPause(NULL, RecordEvent); + assert_true(WatcherRegister(&new_blocking_promise, FileDeletedOptions(path, 300))); + EventWatcherResume(); + + /* Its reference is taken on the remote lane, then a deletion fires while + * the old check is still blocked */ + WaitForFileChecksOk(checks_ok + 1); + assert_int_equal(unlink(path), 0); + WatcherMarkAllDirty(); + HandleEventsSince(fd); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&new_blocking_promise); + assert_int_equal(BlockingStatesDestroyed(), 0); + + /* The old state goes once its check is over, with its event */ + ReleaseBlockedCheck(); + assert_true(WaitUntil(OneBlockingStateDestroyed, 50)); + AssertNoEventHandled(fd, 10); + + EventWatcherFinalize(); + assert_int_equal(BlockingStatesDestroyed(), 1); + RemoveFakeRemoteMount(); + assert_int_equal(rmdir(dir), 0); + alarm(0); +} + +/* A watcher whose checks find an event while switch_events > 0, one per + * check */ +static int switch_events = 0; + +static WatcherCheckResult SwitchCheck(ARG_UNUSED void *state) +{ + int left = __atomic_load_n(&switch_events, __ATOMIC_SEQ_CST); + while (left > 0) + { + if (__atomic_compare_exchange_n(&switch_events, &left, left - 1, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) + { + return WATCHER_CHECK_EVENT; + } + } + return WATCHER_CHECK_NO_EVENT; +} + +static bool SwitchEventsObserved(void) +{ + return (__atomic_load_n(&switch_events, __ATOMIC_SEQ_CST) == 0); +} + +/** Makes the next check of the switch watcher find an event, and waits up to + * 3 s for a check to have found it */ +static void SwitchEvent(void) +{ + __atomic_store_n(&switch_events, 1, __ATOMIC_SEQ_CST); + WatcherMarkAllDirty(); + assert_true(WaitUntil(SwitchEventsObserved, 30)); +} + +static WatcherOptions *SwitchOptions(void) +{ + static WatcherOptions opt; + opt = (WatcherOptions) { + .state = xcalloc(1, 1), + .check_callback = SwitchCheck, + .destroy_state = free, + .interval = 300, + }; + return &opt; +} + +static const Promise switch_promise = { + .promiser = "switch-event", + .org_pp = &unexpanded_promise, + .parent_section = &test_section, +}; + +/** Re-reads the policy with an identical switch watcher, i.e. a new + * unexpanded promise but the same expanded one, and returns the latter */ +static void ReReadWithIdenticalSwitchWatcher(const Promise *unexpanded, Promise *expanded) +{ + *expanded = (Promise) { + .promiser = "switch-event", + .org_pp = unexpanded, + .parent_section = &test_section, + }; + EventWatcherPause(NULL, RecordEvent); + assert_true(WatcherRegister(expanded, SwitchOptions())); + EventWatcherResume(); +} + +/** Starts the event watcher with the switch watcher. Returns the fd. */ +static int StartWithSwitchWatcher(void) +{ + __atomic_store_n(&switch_events, 0, __ATOMIC_SEQ_CST); + ResetRecordedEvents(); + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&switch_promise, SwitchOptions())); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + EventWatcherWakeUp(); + return fd; +} + +/** An event observed while the bundles of the previous one haven't run yet is + * held, and runs once they have */ +static void test_event_held_while_pending_runs_after(void) +{ + alarm(60); + const int fd = StartWithSwitchWatcher(); + + SwitchEvent(); + assert_true(WaitForQueuedEvent(fd, 20)); + SwitchEvent(); /* held, as the first one is pending */ + HandleEvents(fd); + assert_int_equal(handled_event_count, 1); + + HandleEventsUntil(fd, 2, 50); + AssertNoEventHandled(fd, 5); + + EventWatcherFinalize(); + alarm(0); +} + +/** So it does if the policy is re-read before: the held event is in the + * state the identical watcher of the new policy shares, and runs with its + * promise */ +static void test_event_held_while_pending_survives_reread(void) +{ + alarm(60); + const int fd = StartWithSwitchWatcher(); + + SwitchEvent(); + assert_true(WaitForQueuedEvent(fd, 20)); + SwitchEvent(); /* held, as the first one is pending */ + + /* Pausing runs the first one */ + static const Promise new_unexpanded_promise; + Promise new_switch_promise; + ReReadWithIdenticalSwitchWatcher(&new_unexpanded_promise, &new_switch_promise); + assert_int_equal(handled_event_count, 1); + AssertLastEventFor(&switch_promise); + + HandleEventsUntil(fd, 2, 50); + AssertLastEventFor(&new_switch_promise); + AssertNoEventHandled(fd, 5); + + EventWatcherFinalize(); + alarm(0); +} + +/** The limit on how often the bundles run is kept across a policy re-read: + * re-reading doesn't let a watcher that hit it run again right away. The + * event held meanwhile runs once the delay is over. */ +static void test_trigger_limit_survives_reread(void) +{ + alarm(60); + const int fd = StartWithSwitchWatcher(); + + /* MAX_TRIGGERS_PER_WINDOW (10) runs in a row */ + for (int i = 1; i <= 10; i++) + { + SwitchEvent(); + HandleEventsUntil(fd, i, 50); + } + + /* The next one is delayed by MIN_TRIGGER_BACKOFF_SECS (10) */ + const LogLevel old_level = LogGetGlobalLevel(); + LogSetGlobalLevel(LOG_LEVEL_ERR); + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + SwitchEvent(); + AssertNoEventHandled(fd, 10); + LogSetGlobalLevel(old_level); + + static const Promise new_unexpanded_promise; + Promise new_switch_promise; + ReReadWithIdenticalSwitchWatcher(&new_unexpanded_promise, &new_switch_promise); + AssertNoEventHandled(fd, 20); + + HandleEventsUntil(fd, 11, 100); + AssertLastEventFor(&new_switch_promise); + assert_true(ElapsedSecs(&start) >= 9); + + EventWatcherFinalize(); + alarm(0); +} + +/** Finalizing doesn't hang on a remote check that never returns: the lane is + * left behind with what it uses. Last, as the registry is leaked. */ +static void test_finalize_with_stuck_remote_check(void) +{ + alarm(60); + FakeRemoteMount("/remote"); + ResetBlockedCheck(); + ResetRecordedEvents(); + + WatcherRegistryInitialize(); + assert_true(WatcherRegister(&blocking_promise, BlockingOptions("/remote/blocking", 1))); + int fd = -1; + assert_true(EventWatcherInitialize(&fd)); + WaitForBlockedCheck(); + + const LogLevel old_level = LogGetGlobalLevel(); + LogSetGlobalLevel(LOG_LEVEL_CRIT); + struct timespec start; + clock_gettime(CLOCK_MONOTONIC, &start); + EventWatcherFinalize(); + assert_true(ElapsedSecs(&start) < 8); /* the 5 s timeout, not forever */ + LogSetGlobalLevel(old_level); + assert_int_equal(BlockingStatesDestroyed(), 0); + + /* The left behind lane may finish later, on what it was left */ + ReleaseBlockedCheck(); + sleep(1); + WatchLocationSetMountTable(NULL); + alarm(0); +} + int main() { PRINT_TEST_BANNER(); + + /* Everything is local, unless a test fakes a remote mount */ + WatchLocationSetMountTable(""); + const UnitTest tests[] = { unit_test(test_registry_initialize_finalize), @@ -611,6 +1694,26 @@ int main() unit_test(test_event_watcher_several_events), unit_test(test_event_watcher_check_failure_reported_once), unit_test(test_watcher_register_check_failure), + unit_test(test_watcher_check_failure_retried_before_interval), + unit_test(test_event_watcher_fires_at_interval), + unit_test(test_watchers_report_written_when_changed_removed_on_finalize), + unit_test(test_blocked_check_does_not_block_registry), + unit_test(test_blocked_check_watcher_removed), + unit_test(test_blocked_check_state_taken_over), + unit_test(test_blocked_check_state_taken_over_after_two_rereads), + unit_test(test_blocked_check_finalized_before_handoff), + unit_test(test_event_held_while_pending_runs_after), + unit_test(test_event_held_while_pending_survives_reread), + unit_test(test_trigger_limit_survives_reread), + unit_test(test_mountinfo_path_classification), + unit_test(test_mountinfo_fs_types), + unit_test(test_path_is_unc), + unit_test(test_watch_target_is_remote), + unit_test(test_remote_watcher_reference_taken_on_its_lane), + unit_test(test_blocked_remote_check_does_not_delay_local), + unit_test(test_watcher_moved_to_other_lane_starts_afresh), + /* Last, see there */ + unit_test(test_finalize_with_stuck_remote_check), }; return run_tests(tests);