diff --git a/src/runner.c b/src/runner.c
index f19338e..deeef80 100644
--- a/src/runner.c
+++ b/src/runner.c
@@ -329,6 +329,12 @@ schedule_exec(struct lmapd *lmapd, struct schedule *schedule)
// lmap_dbg("executing schedule '%s'", schedule->name);
+ /* avoid leftover data (possibly due to a crash) from
+ * previous runs of an action. */
+ for (act = schedule->actions; act; act = act->next) {
+ (void) lmapd_workspace_action_clean(lmapd, act);
+ }
+
event_base_gettimeofday_cached(lmapd->base, &t);
switch (schedule->mode) {
@@ -528,7 +534,7 @@ void
lmapd_cleanup(struct lmapd *lmapd)
{
pid_t pid;
- int status, failed;
+ int status, failed, succeeded;
struct lmap *lmap;
struct timeval t;
struct action *action;
@@ -619,7 +625,9 @@ lmapd_cleanup(struct lmapd *lmapd)
/*
* Change schedule state back to enabled if all actions have
- * left the running state.
+ * left the running state. If at least one action was executed
+ * and every action returned success, clean up the schedule
+ * input processing queue.
*/
if (schedule->state == LMAP_SCHEDULE_STATE_RUNNING) {
@@ -627,17 +635,24 @@ lmapd_cleanup(struct lmapd *lmapd)
if (schedule->cnt_active_suppressions) {
schedule->state = LMAP_SCHEDULE_STATE_SUPPRESSED;
}
- failed = 0;
+ succeeded = failed = 0;
for (action = schedule->actions; action; action = action->next) {
if (action->state == LMAP_ACTION_STATE_RUNNING) {
schedule->state = LMAP_SCHEDULE_STATE_RUNNING;
}
if (action->last_status) {
- failed++;
+ failed = 1;
+ } else {
+ succeeded = 1;
}
}
- if (schedule->state != LMAP_SCHEDULE_STATE_RUNNING && failed) {
- schedule->cnt_failures++;
+ if (schedule->state != LMAP_SCHEDULE_STATE_RUNNING) {
+ if (failed) {
+ schedule->cnt_failures++;
+ } else if (succeeded) {
+ /* there was at least one action, and none failed */
+ lmapd_workspace_schedule_clean(lmapd, schedule);
+ }
}
}
}
@@ -694,7 +709,8 @@ execute_cb(struct lmapd *lmapd, struct event *event)
event_base_gettimeofday_cached(lmapd->base, &t);
sched->cycle_number = (t.tv_sec / event->cycle_interval) * event->cycle_interval;
}
-
+
+ lmapd_workspace_schedule_move(lmapd, sched);
schedule_exec(lmapd, sched);
if (event->type == LMAP_EVENT_TYPE_ONE_OFF
|| event->type == LMAP_EVENT_TYPE_IMMEDIATE
diff --git a/src/workspace.c b/src/workspace.c
index 389c16b..4f99b23 100644
--- a/src/workspace.c
+++ b/src/workspace.c
@@ -15,14 +15,17 @@
* along with lmapd. If not, see .
*/
-#define _XOPEN_SOURCE 500
+#define _XOPEN_SOURCE 700
#define _POSIX_C_SOURCE 200809L
+#define _GNU_SOURCE 1 /* open(O_DIRECTORY), dirfd() prototype */
#include
#include
#include
#include
#include
+#include
+#include
#include
#include
#include
@@ -37,13 +40,21 @@
#include "csv.h"
#include "workspace.h"
+/* incoming schedule queue name, must start with _ */
+#define LMAPD_QUEUE_INCOMING_NAME "_incoming"
+
static const char delimiter = ';';
/**
* @brief Create a safe filesystem name
*
* Creates a safe filesystem name. Unsafe characters are %-encoded if
- * necessary.
+ * necessary. It ensures the filename does not start with [._] to
+ * avoid creating hidden files, and to give lmapd a private namespace
+ * to work with (anything starting with "_").
+ *
+ * Note: as a side-effect, does not allow filenames to start with a
+ * few other characters, either, and will %-escape them instead.
*
* @param name file system name
* @return pointer to a safe filesystem name (static buffer)
@@ -58,7 +69,7 @@ mksafe(const char *name)
static char save_name[NAME_MAX];
for (i = 0, j = 0; name[i] && j < NAME_MAX-1; i++) {
- if (isalnum(name[i]) || strchr(safe, name[i])) {
+ if (isalnum(name[i]) || (i > 0 && strchr(safe, name[i]))) {
save_name[j++] = name[i];
} else {
/* %-escape the char if there is enough space left */
@@ -223,12 +234,191 @@ lmapd_workspace_update(struct lmapd *lmapd)
return ret;
}
+/**
+ * @brief Clean the workspace of an schedule
+ *
+ * Function to clean the workspace of an schedule, by removing
+ * the processing queue (all files in the base schedule directory),
+ * it leaves directories and files starting with "_" untouched.
+ *
+ * @param lmapd pointer to the struct lmapd
+ * @param schedule pointer to the struct schedule
+ * @return 0 on success, -1 on error
+ */
+int
+lmapd_workspace_schedule_clean(struct lmapd *lmapd, struct schedule *schedule)
+{
+ int ret = 0;
+ struct dirent *dp;
+ struct stat st;
+ DIR *dfd;
+
+ assert(lmapd);
+ (void) lmapd;
+
+ if (!schedule || !schedule->workspace) {
+ return 0;
+ }
+
+ dfd = opendir(schedule->workspace);
+ if (!dfd) {
+ lmap_err("failed to open directory '%s'", schedule->workspace);
+ return -1;
+ }
+
+ while ((dp = readdir(dfd)) != NULL) {
+ if (dp->d_name[0] == '_') {
+ continue;
+ }
+ if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW)
+ || S_ISDIR(st.st_mode)) {
+ continue;
+ }
+ if (unlinkat(dirfd(dfd), dp->d_name, 0)) {
+ lmap_err("failed to remove '%s/%s'", schedule->workspace, dp->d_name);
+ ret = -1;
+ }
+ }
+ (void) closedir(dfd);
+
+ return ret;
+}
+
+/**
+ * @brief Move the workspace incoming queue of an schedule
+ *
+ * Function to move the contents of the incoming special
+ * queue of an schedule to the active input queue.
+ *
+ * Only complete queue entries (i.e. those with both .data
+ * and .meta files) are moved.
+ *
+ * @param lmapd pointer to the struct lmapd
+ * @param schedule pointer to the struct schedule
+ * @return 0 on success, -1 on error
+ */
+
+int
+lmapd_workspace_schedule_move(struct lmapd *lmapd, struct schedule *schedule)
+{
+ int ret = -1;
+ char oldfilepath[PATH_MAX];
+ struct dirent *dp;
+ DIR *dfd;
+ struct stat st;
+
+ int dirfd_dest = -1;
+ char *sdata = NULL, *s;
+
+ assert(lmapd);
+ (void) lmapd;
+
+ if (!schedule || !schedule->workspace) {
+ return 0;
+ }
+
+ const char * const newfilepath = schedule->workspace;
+
+ errno = 0;
+ do {
+ dirfd_dest = open(newfilepath, O_DIRECTORY | O_RDONLY);
+ } while (dirfd_dest == -1 && (errno == EAGAIN || errno == EINTR));
+ if (dirfd_dest == -1) {
+ lmap_err("failed to open directory '%s': %s",
+ newfilepath, strerror(errno));
+ return -1;
+ }
+
+ snprintf(oldfilepath, sizeof(oldfilepath), "%s/" LMAPD_QUEUE_INCOMING_NAME,
+ schedule->workspace);
+ dfd = opendir(oldfilepath);
+ if (!dfd) {
+ lmap_err("failed to open directory '%s': %s",
+ oldfilepath, strerror(errno));
+ goto err_exit;
+ }
+
+ sdata = NULL;
+ while ((dp = readdir(dfd)) != NULL) {
+ /* skip ., .., hidden files/directories */
+ if (dp->d_name[0] == '.') {
+ continue;
+ }
+ /* is it the .meta file ? */
+ s = strrchr(dp->d_name, '.');
+ if (s && !strcmp(".meta", s)) {
+ free(sdata);
+ sdata = strdup(dp->d_name);
+ if (!sdata)
+ break; /* abort scan */
+ s = strrchr(sdata, '.');
+ if (!s)
+ break; /* should *NEVER* happen */
+ s++;
+ strcpy(s, "data"); /* strlen("data") == strlen("meta") */
+
+ if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW)
+ || !S_ISREG(st.st_mode)) {
+ continue; /* "meta" is not a regular file? skip this pair */
+ }
+ /* "meta" *is* there, "data" might not be */
+ if (fstatat(dirfd(dfd), sdata, &st, AT_SYMLINK_NOFOLLOW)
+ || !S_ISREG(st.st_mode)) {
+ continue; /* "data" is not a regular file, or not there yet, skip this pair */
+ }
+ if (linkat(dirfd(dfd), sdata, dirfd_dest, sdata, 0)) {
+ lmap_err("failed to move %s from %s to %s: %s",
+ sdata, oldfilepath, newfilepath, strerror(errno));
+ continue;
+ }
+ if (linkat(dirfd(dfd), dp->d_name, dirfd_dest, dp->d_name, 0)) {
+ lmap_err("failed to move %s from %s to %s: %s",
+ dp->d_name, oldfilepath, newfilepath, strerror(errno));
+ /* rollback first linkat() */
+ if (unlinkat(dirfd_dest, sdata, 0))
+ lmap_err("Could not rollback move of '%s/%s': %s",
+ oldfilepath, sdata, strerror(errno));
+ break;
+ }
+ /* unlink from source dir to complete the move operation, do not
+ * short-circuit -- if we unlink either one, we already avoid
+ * double processing them */
+ if (unlinkat(dirfd(dfd), dp->d_name, 0)) {
+ lmap_wrn("failed to unlink %s from incoming queue: %s",
+ dp->d_name, strerror(errno));
+ }
+ if (unlinkat(dirfd(dfd), sdata, 0)) {
+ lmap_wrn("failed to unlink %s from incoming queue: %s",
+ sdata, strerror(errno));
+ }
+ }
+ }
+ ret = 0;
+
+err_exit:
+ free(sdata);
+ if (dfd) {
+ (void) closedir(dfd);
+ }
+ if (dirfd_dest != -1) {
+ (void) close(dirfd_dest);
+ }
+
+ return ret;
+}
+
/**
* @brief Clean the workspace of an action
*
* Function to clean the workspace of an action by removing everything
* in the workspace directory.
*
+ * We preserve files and direct subdirectories (but not files or
+ * directories inside subdirectories) starting with "_", so that actions
+ * can have a private namespace for work that, while not guaranteed to
+ * last across lmapd config updates and restarts, keeps state from one
+ * schedule execution to the next.
+ *
* @param lmapd pointer to the struct lmapd
* @param action pointer to the struct action
* @return 0 on success, -1 on error
@@ -259,6 +449,9 @@ lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action)
if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) {
continue;
}
+ if (dp->d_name[0] == '_') {
+ continue;
+ }
snprintf(filepath, sizeof(filepath), "%s/%s",
action->workspace, dp->d_name);
if (remove_all(filepath) != 0) {
@@ -275,7 +468,17 @@ lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action)
* @brief Move the workspace of an action
*
* Function to move the workspace of an action to a destination
- * schedule.
+ * schedule. The action output files are moved to the destination
+ * schedule's incoming special folder.
+ *
+ * Note: all files in the action workspace are moved, except for
+ * hidden files, or those starting with "_".
+ *
+ * Should the destination schedule be the action's own schedule,
+ * move its output files to its own schedule's *active* (processing)
+ * incoming queue instead, where it will be immediately available
+ * for consumption (e.g. by the next action in a sequential execution
+ * mode).
*
* @param lmapd pointer to the struct lmapd
* @param schedule pointer to the struct schedule
@@ -291,8 +494,10 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule,
int ret = 0;
char oldfilepath[PATH_MAX];
char newfilepath[PATH_MAX];
+ const char *newfileformat;
struct dirent *dp;
DIR *dfd;
+ struct stat st;
assert(lmapd);
(void) lmapd;
@@ -303,6 +508,13 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule,
return 0;
}
+ if (destination != schedule) {
+ newfileformat = "%s/" LMAPD_QUEUE_INCOMING_NAME "/%s";
+ } else {
+ /* Special case an action moving to its own schedule */
+ newfileformat = "%s/%s";
+ }
+
dfd = opendir(action->workspace);
if (!dfd) {
lmap_err("failed to open '%s'", action->workspace);
@@ -310,13 +522,20 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule,
}
while ((dp = readdir(dfd)) != NULL) {
- if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) {
+ /* we only "move" files, never directories or other inode types */
+ if (fstatat(dirfd(dfd), dp->d_name, &st, AT_SYMLINK_NOFOLLOW)
+ || !S_ISREG(st.st_mode)) {
+ continue;
+ }
+ if (dp->d_name[0] == '_' || dp->d_name[0] == '.') {
continue;
}
+ /* we don't need to special case . and .. directories due
+ * to the above */
snprintf(oldfilepath, sizeof(oldfilepath), "%s/%s",
action->workspace, dp->d_name);
- snprintf(newfilepath, sizeof(newfilepath), "%s/%s",
- destination->workspace, dp->d_name);
+ snprintf(newfilepath, sizeof(newfilepath), newfileformat,
+ destination->workspace, dp->d_name);
if (link(oldfilepath, newfilepath) < 0) {
lmap_err("failed to move '%s' to '%s'", oldfilepath, newfilepath);
ret = -1;
@@ -332,7 +551,7 @@ lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule,
*
* Function to create the workspace folders for schedules and their
* actions. Actions store results before in their workspace sending
- * them to the destination schedule.
+ * them to the destination schedule's incoming special folder.
*
* @param lmapd pointer to struct lmapd
* @return 0 on success, -1 on error
@@ -377,6 +596,14 @@ lmapd_workspace_init(struct lmapd *lmapd)
}
lmap_action_set_workspace(act, filepath);
}
+
+ /* create incoming directory */
+ snprintf(filepath, sizeof(filepath), "%s/" LMAPD_QUEUE_INCOMING_NAME,
+ sched->workspace);
+ if (mkdir(filepath, 0700) < 0 && errno != EEXIST) {
+ lmap_err("failed to mkdir '%s'", filepath);
+ ret = -1;
+ }
}
return ret;
diff --git a/src/workspace.h b/src/workspace.h
index f5f0c69..be864ba 100644
--- a/src/workspace.h
+++ b/src/workspace.h
@@ -27,6 +27,8 @@ extern int lmapd_workspace_update(struct lmapd *lmapd);
extern int lmapd_workspace_action_clean(struct lmapd *lmapd, struct action *action);
extern int lmapd_workspace_action_move(struct lmapd *lmapd, struct schedule *schedule, struct action *action, struct schedule *destination);
+extern int lmapd_workspace_schedule_move(struct lmapd *lmapd, struct schedule *schedule);
+extern int lmapd_workspace_schedule_clean(struct lmapd *lmapd, struct schedule *schedule);
extern int lmapd_workspace_action_open_data(struct schedule *schedule, struct action *action, int flags);