diff --git a/src/runner.c b/src/runner.c index f19338e..deeef80 100644 --- a/src/runner.c +++ b/src/runner.c @@ -329,6 +329,12 @@ schedule_exec(struct lmapd *lmapd, struct schedule *schedule) // lmap_dbg("executing schedule '%s'", schedule->name); + /* avoid leftover data (possibly due to a crash) from + * previous runs of an action. */ + for (act = schedule->actions; act; act = act->next) { + (void) lmapd_workspace_action_clean(lmapd, act); + } + event_base_gettimeofday_cached(lmapd->base, &t); switch (schedule->mode) { @@ -528,7 +534,7 @@ void lmapd_cleanup(struct lmapd *lmapd) { pid_t pid; - int status, failed; + int status, failed, succeeded; struct lmap *lmap; struct timeval t; struct action *action; @@ -619,7 +625,9 @@ lmapd_cleanup(struct lmapd *lmapd) /* * Change schedule state back to enabled if all actions have - * left the running state. + * left the running state. If at least one action was executed + * and every action returned success, clean up the schedule + * input processing queue. */ if (schedule->state == LMAP_SCHEDULE_STATE_RUNNING) { @@ -627,17 +635,24 @@ lmapd_cleanup(struct lmapd *lmapd) if (schedule->cnt_active_suppressions) { schedule->state = LMAP_SCHEDULE_STATE_SUPPRESSED; } - failed = 0; + succeeded = failed = 0; for (action = schedule->actions; action; action = action->next) { if (action->state == LMAP_ACTION_STATE_RUNNING) { schedule->state = LMAP_SCHEDULE_STATE_RUNNING; } if (action->last_status) { - failed++; + failed = 1; + } else { + succeeded = 1; } } - if (schedule->state != LMAP_SCHEDULE_STATE_RUNNING && failed) { - schedule->cnt_failures++; + if (schedule->state != LMAP_SCHEDULE_STATE_RUNNING) { + if (failed) { + schedule->cnt_failures++; + } else if (succeeded) { + /* there was at least one action, and none failed */ + lmapd_workspace_schedule_clean(lmapd, schedule); + } } } } @@ -694,7 +709,8 @@ execute_cb(struct lmapd *lmapd, struct event *event) event_base_gettimeofday_cached(lmapd->base, &t); sched->cycle_number = (t.tv_sec / event->cycle_interval) * event->cycle_interval; } - + + lmapd_workspace_schedule_move(lmapd, sched); schedule_exec(lmapd, sched); if (event->type == LMAP_EVENT_TYPE_ONE_OFF || event->type == LMAP_EVENT_TYPE_IMMEDIATE diff --git a/src/workspace.c b/src/workspace.c index 389c16b..4f99b23 100644 --- a/src/workspace.c +++ b/src/workspace.c @@ -15,14 +15,17 @@ * along with lmapd. If not, see . */ -#define _XOPEN_SOURCE 500 +#define _XOPEN_SOURCE 700 #define _POSIX_C_SOURCE 200809L +#define _GNU_SOURCE 1 /* open(O_DIRECTORY), dirfd() prototype */ #include #include #include #include #include +#include +#include #include #include #include @@ -37,13 +40,21 @@ #include "csv.h" #include "workspace.h" +/* incoming schedule queue name, must start with _ */ +#define LMAPD_QUEUE_INCOMING_NAME "_incoming" + static const char delimiter = ';'; /** * @brief Create a safe filesystem name * * Creates a safe filesystem name. Unsafe characters are %-encoded if - * necessary. + * necessary. It ensures the filename does not start with [._] to + * avoid creating hidden files, and to give lmapd a private namespace + * to work with (anything starting with "_"). + * + * Note: as a side-effect, does not allow filenames to start with a + * few other characters, either, and will %-escape them instead. * * @param name file system name * @return pointer to a safe filesystem name (static buffer) @@ -58,7 +69,7 @@ mksafe(const char *name) static char save_name[NAME_MAX]; for (i = 0, j = 0; name[i] && j < NAME_MAX-1; i++) { - if (isalnum(name[i]) || strchr(safe, name[i])) { + if (isalnum(name[i]) || (i > 0 && strchr(safe, name[i]))) { save_name[j++] = name[i]; } else { /* %-escape the char if there is enough space left */ @@ -223,12 +234,191 @@ lmapd_workspace_update(struct lmapd *lmapd) return ret; } +/** + * @brief Clean the workspace of an schedule + * + * Function to clean the workspace of an schedule, by removing + * the processing queue (all files in the base schedule directory), + * it leaves directories and files starting with "_" untouched. + * + * @param lmapd pointer to the struct lmapd + * @param schedule pointer to the struct schedule + * @return 0 on success, -1 on error + */ +int +lmapd_workspace_schedule_clean(struct lmapd *lmapd, struct schedule *schedule) +{ + int ret = 0; + struct dirent *dp; + struct stat st; + DIR *dfd; + + assert(lmapd); + (void) lmapd; + + if (!schedule || !schedule->workspace) { + return 0; + } + + dfd = opendir(schedule->workspace); + if (!dfd) { + lmap_err("failed to open directory '%s'", schedule->workspace); + return -1; + } + + while ((dp = readdir(dfd)) != NULL) { + if (dp->d_name[0] == '_') { + continue; + } + if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW) + || S_ISDIR(st.st_mode)) { + continue; + } + if (unlinkat(dirfd(dfd), dp->d_name, 0)) { + lmap_err("failed to remove '%s/%s'", schedule->workspace, dp->d_name); + ret = -1; + } + } + (void) closedir(dfd); + + return ret; +} + +/** + * @brief Move the workspace incoming queue of an schedule + * + * Function to move the contents of the incoming special + * queue of an schedule to the active input queue. + * + * Only complete queue entries (i.e. those with both .data + * and .meta files) are moved. + * + * @param lmapd pointer to the struct lmapd + * @param schedule pointer to the struct schedule + * @return 0 on success, -1 on error + */ + +int +lmapd_workspace_schedule_move(struct lmapd *lmapd, struct schedule *schedule) +{ + int ret = -1; + char oldfilepath[PATH_MAX]; + struct dirent *dp; + DIR *dfd; + struct stat st; + + int dirfd_dest = -1; + char *sdata = NULL, *s; + + assert(lmapd); + (void) lmapd; + + if (!schedule || !schedule->workspace) { + return 0; + } + + const char * const newfilepath = schedule->workspace; + + errno = 0; + do { + dirfd_dest = open(newfilepath, O_DIRECTORY | O_RDONLY); + } while (dirfd_dest == -1 && (errno == EAGAIN || errno == EINTR)); + if (dirfd_dest == -1) { + lmap_err("failed to open directory '%s': %s", + newfilepath, strerror(errno)); + return -1; + } + + snprintf(oldfilepath, sizeof(oldfilepath), "%s/" LMAPD_QUEUE_INCOMING_NAME, + schedule->workspace); + dfd = opendir(oldfilepath); + if (!dfd) { + lmap_err("failed to open directory '%s': %s", + oldfilepath, strerror(errno)); + goto err_exit; + } + + sdata = NULL; + while ((dp = readdir(dfd)) != NULL) { + /* skip ., .., hidden files/directories */ + if (dp->d_name[0] == '.') { + continue; + } + /* is it the .meta file ? */ + s = strrchr(dp->d_name, '.'); + if (s && !strcmp(".meta", s)) { + free(sdata); + sdata = strdup(dp->d_name); + if (!sdata) + break; /* abort scan */ + s = strrchr(sdata, '.'); + if (!s) + break; /* should *NEVER* happen */ + s++; + strcpy(s, "data"); /* strlen("data") == strlen("meta") */ + + if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW) + || !S_ISREG(st.st_mode)) { + continue; /* "meta" is not a regular file? skip this pair */ + } + /* "meta" *is* there, "data" might not be */ + if (fstatat(dirfd(dfd), sdata, &st, AT_SYMLINK_NOFOLLOW) + || !S_ISREG(st.st_mode)) { + continue; /* "data" is not a regular file, or not there yet, skip this pair */ + } + if (linkat(dirfd(dfd), sdata, dirfd_dest, sdata, 0)) { + lmap_err("failed to move %s from %s to %s: %s", + sdata, oldfilepath, newfilepath, strerror(errno)); + continue; + } + if (linkat(dirfd(dfd), dp->d_name, dirfd_dest, dp->d_name, 0)) { + lmap_err("failed to move %s from %s to %s: %s", + dp->d_name, oldfilepath, newfilepath, strerror(errno)); + /* rollback first linkat() */ + if (unlinkat(dirfd_dest, sdata, 0)) + lmap_err("Could not rollback move of '%s/%s': %s", + oldfilepath, sdata, strerror(errno)); + break; + } + /* unlink from source dir to complete the move operation, do not + * short-circuit -- if we unlink either one, we already avoid + * double processing them */ + if (unlinkat(dirfd(dfd), dp->d_name, 0)) { + lmap_wrn("failed to unlink %s from incoming queue: %s", + dp->d_name, strerror(errno)); + } + if (unlinkat(dirfd(dfd), sdata, 0)) { + lmap_wrn("failed to unlink %s from incoming queue: %s", + sdata, strerror(errno)); + } + } + } + ret = 0; + +err_exit: + free(sdata); + if (dfd) { + (void) closedir(dfd); + } + if (dirfd_dest != -1) { + (void) close(dirfd_dest); + } + + return ret; +} + /** * @brief Clean the workspace of an action * * Function to clean the workspace of an action by removing everything * in the workspace directory. * + * We preserve files and direct subdirectories (but not files or + * directories inside subdirectories) starting with "_", so that actions + * can have a private namespace for work that, while not guaranteed to + * last across lmapd config updates and restarts, keeps state from one + * schedule execution to the next. + * * @param lmapd pointer to the struct lmapd * @param action pointer to the struct action * @return 0 on success, -1 on error @@ -259,6 +449,9 @@ lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action) if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) { continue; } + if (dp->d_name[0] == '_') { + continue; + } snprintf(filepath, sizeof(filepath), "%s/%s", action->workspace, dp->d_name); if (remove_all(filepath) != 0) { @@ -275,7 +468,17 @@ lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action) * @brief Move the workspace of an action * * Function to move the workspace of an action to a destination - * schedule. + * schedule. The action output files are moved to the destination + * schedule's incoming special folder. + * + * Note: all files in the action workspace are moved, except for + * hidden files, or those starting with "_". + * + * Should the destination schedule be the action's own schedule, + * move its output files to its own schedule's *active* (processing) + * incoming queue instead, where it will be immediately available + * for consumption (e.g. by the next action in a sequential execution + * mode). * * @param lmapd pointer to the struct lmapd * @param schedule pointer to the struct schedule @@ -291,8 +494,10 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, int ret = 0; char oldfilepath[PATH_MAX]; char newfilepath[PATH_MAX]; + const char *newfileformat; struct dirent *dp; DIR *dfd; + struct stat st; assert(lmapd); (void) lmapd; @@ -303,6 +508,13 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, return 0; } + if (destination != schedule) { + newfileformat = "%s/" LMAPD_QUEUE_INCOMING_NAME "/%s"; + } else { + /* Special case an action moving to its own schedule */ + newfileformat = "%s/%s"; + } + dfd = opendir(action->workspace); if (!dfd) { lmap_err("failed to open '%s'", action->workspace); @@ -310,13 +522,20 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, } while ((dp = readdir(dfd)) != NULL) { - if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) { + /* we only "move" files, never directories or other inode types */ + if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW) + || !S_ISREG(st.st_mode)) { + continue; + } + if (dp->d_name[0] == '_' || dp->d_name[0] == '.') { continue; } + /* we don't need to special case . and .. directories due + * to the above */ snprintf(oldfilepath, sizeof(oldfilepath), "%s/%s", action->workspace, dp->d_name); - snprintf(newfilepath, sizeof(newfilepath), "%s/%s", - destination->workspace, dp->d_name); + snprintf(newfilepath, sizeof(newfilepath), newfileformat, + destination->workspace, dp->d_name); if (link(oldfilepath, newfilepath) < 0) { lmap_err("failed to move '%s' to '%s'", oldfilepath, newfilepath); ret = -1; @@ -332,7 +551,7 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, * * Function to create the workspace folders for schedules and their * actions. Actions store results before in their workspace sending - * them to the destination schedule. + * them to the destination schedule's incoming special folder. * * @param lmapd pointer to struct lmapd * @return 0 on success, -1 on error @@ -377,6 +596,14 @@ lmapd_workspace_init(struct lmapd *lmapd) } lmap_action_set_workspace(act, filepath); } + + /* create incoming directory */ + snprintf(filepath, sizeof(filepath), "%s/" LMAPD_QUEUE_INCOMING_NAME, + sched->workspace); + if (mkdir(filepath, 0700) < 0 && errno != EEXIST) { + lmap_err("failed to mkdir '%s'", filepath); + ret = -1; + } } return ret; diff --git a/src/workspace.h b/src/workspace.h index f5f0c69..be864ba 100644 --- a/src/workspace.h +++ b/src/workspace.h @@ -27,6 +27,8 @@ extern int lmapd_workspace_update(struct lmapd *lmapd); extern int lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action); extern int lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, struct action *action, struct schedule *destination); +extern int lmapd_workspace_schedule_move(struct lmapd *lmapd, struct schedule *schedule); +extern int lmapd_workspace_schedule_clean(struct lmapd *lmapd, struct schedule *schedule); extern int lmapd_workspace_action_open_data(struct schedule *schedule, struct action *action, int flags);