fence_sanlock/fence_sanlock.8 | 85 ++++++++++++++++++++++++++++++++++++++---
fence_sanlock/fence_sanlockd.8 | 4 +
fence_sanlock/fence_sanlockd.c | 68 ++++++++++++++++++++++++++++++++
src/cmd.c | 2
src/helper.c | 2
src/resource.c | 29 +++++--------
src/sanlock_internal.h | 4 +
src/sanlock_resource.h | 20 ++++++---
8 files changed, 182 insertions(+), 32 deletions(-)
New commits:
commit 4c058f277ad2dbd852cd3087f48edf4562de0bb3
Author: David Teigland <teigland(a)redhat.com>
Date: Wed Apr 10 16:01:27 2013 -0500
sanlock/fence_sanlock: request modes FORCE and GRACEFUL
FORCE is new name for KILL_PID.
GRACEFUL is killpath (or SIGTERM of no killpath is set)
SIGUSR1 is removed; a killpath prog should send the signal
fence_sanlock is converted to use killpath to send SIGUSR1.
The killpath program is "fence_sanlockd -1" (-1 is new).
Change sanlock helper to use execvp so we don't need to
know the fence_sanlockd path.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/fence_sanlock/fence_sanlock.8 b/fence_sanlock/fence_sanlock.8
index d9e41be..66f0912 100644
--- a/fence_sanlock/fence_sanlock.8
+++ b/fence_sanlock/fence_sanlock.8
@@ -1,4 +1,4 @@
-.TH FENCE_SANLOCK 8 2012-09-26
+.TH FENCE_SANLOCK 8 2013-05-02
.SH NAME
fence_sanlock \- fence agent using watchdog and shared storage leases
@@ -166,7 +166,11 @@ See \-i
.SH FILES
-Example cluster.conf configuration for fence_sanlock:
+Example cluster.conf configuration for fence_sanlock.
+.br
+(For cman based clusters in which fenced runs agents.)
+.br
+Also see cluster.conf(5), fenced(8), fence_node(8).
.nf
<clusternode name="node01" nodeid="1">
@@ -194,10 +198,81 @@ Example cluster.conf configuration for fence_sanlock:
<fencedevice name="wd" agent="fence_sanlock" path="/dev/fence/leases"/>
.fi
+.PP
+
+Example dlm.conf configuration for fence_sanlock.
+.br
+(For non-cman based clusters in which dlm_controld runs agents.)
+.br
+Also see dlm.conf(5), dlm_controld(8).
+
+.nf
+device wd /usr/sbin/fence_sanlock path=/dev/fence/leases
+connect wd node=1 host_id=1
+connect wd node=2 host_id=2
+unfence wd
+.fi
+
+.SH TEST
+
+To test fence_sanlock directly, without clustering:
+
+.nf
+1. Initialize storage
+
+node1: create 1G lv on shared storage /dev/fence/leases
+node1: fence_sanlock -o sanlock_init -p /dev/fence/leases
+
+2. Start services
+
+node1: service fence_sanlockd start
+node2: service fence_sanlockd start
+
+3. Enable fencing
+
+node1: fence_sanlock -o on -p /dev/fence/leases -i 1
+node2: fence_sanlock -o on -p /dev/fence/leases -i 2
+
+This "unfence" step may take a couple minutes.
+
+4. Verify hosts and leases
+
+node1: sanlock status
+s fence:1:/dev/fence/leases:0
+r fence:h1:/dev/fence/leases:1048576:1 p 2465
+
+node2: sanlock status
+s fence:2:/dev/fence/leases:0
+r fence:h2:/dev/fence/leases:2097152:1 p 2366
+
+node1: sanlock host_status
+lockspace fence
+1 timestamp 717
+2 timestamp 678
+
+node2: sanlock host_status
+lockspace fence
+1 timestamp 738
+2 timestamp 678
+
+5. Fence node2
+
+node1: fence_sanlock -o off -p /dev/fence/leases -i 2
+
+This may take a few minutes to return.
+
+When node2 is not dead before fencing, sanlock on node1 will log errors
+about failing to acquire the lease while node2 is still alive. This is
+expected.
+
+6. Success
+
+node1 fence_sanlock should exit 0 after node2 is reset by its watchdog.
+.fi
+
+
.SH SEE ALSO
.BR fence_sanlockd (8),
.BR sanlock (8),
-.BR wdmd (8),
-.BR fence_node (8),
-.BR fenced (8)
+.BR wdmd (8)
diff --git a/fence_sanlock/fence_sanlockd.8 b/fence_sanlock/fence_sanlockd.8
index 33ce051..33f2dc7 100644
--- a/fence_sanlock/fence_sanlockd.8
+++ b/fence_sanlock/fence_sanlockd.8
@@ -30,6 +30,10 @@ for full description.
.B \-s
Send options (p,i) to waiting fence_sanlockd -w.
+.B \-1
+ Send SIGUSR1 to running fence_sanlockd.
+
+
.SH SEE ALSO
.BR fence_sanlock (8),
.BR sanlock (8),
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index aa45736..b52bd7c 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -149,6 +149,36 @@ static int client_add(int fd, void (*workfn)(int ci), void (*deadfn)(int ci))
goto again;
}
+static int read_lockfile(int *pid)
+{
+ char buf[16];
+ int fd, rv;
+
+ sprintf(lockfile_path, "%s/%s.pid", DAEMON_RUN_DIR, prog_name);
+
+ fd = open(lockfile_path, O_RDONLY);
+ if (fd < 0) {
+ log_error("lockfile open error %s: %s",
+ lockfile_path, strerror(errno));
+ return -1;
+ }
+
+ memset(buf, 0, sizeof(buf));
+
+ rv = read(fd, buf, sizeof(buf));
+ if (rv < 0) {
+ log_error("lockfile read error %s: %s",
+ lockfile_path, strerror(errno));
+ close(fd);
+ return -1;
+ }
+
+ *pid = atoi(buf);
+
+ close(fd);
+ return 0;
+}
+
static int lockfile(void)
{
char buf[16];
@@ -352,6 +382,26 @@ static int send_options(void)
return rv;
}
+static int send_signal(int sig)
+{
+ int rv, pid;
+
+ openlog("fence_sanlockd-1", LOG_CONS | LOG_PID, LOG_DAEMON);
+
+ rv = read_lockfile(&pid);
+ if (rv < 0)
+ return rv;
+
+ rv = kill(pid, sig);
+ if (rv < 0) {
+ log_error("kill sig %d pid %d error %d", sig, pid, errno);
+ } else {
+ syslog(LOG_INFO, "sent signal %d to pid %d", sig, pid);
+ }
+
+ return rv;
+}
+
/*
* A running fence_sanlock agent has a pid file we can read.
* We use this to check what host_id it's fencing, so we can
@@ -464,6 +514,7 @@ static void print_usage(void)
printf(" -i <host_id> Local sanlock host_id (1-%d)\n", MAX_HOSTS);
printf(" -w Wait for fence_sanlockd -s to send options (p,i)\n");
printf(" -s Send options (p,i) to waiting fence_sanlockd -w\n");
+ printf(" -1 Send SIGUSR1 to running fence_sanlockd\n");
printf(" -h Print this help, then exit\n");
printf(" -V Print program version information, then exit\n");
}
@@ -476,6 +527,7 @@ int main(int argc, char *argv[])
int poll_timeout;
int sleep_seconds;
int send_opts = 0, wait_opts = 0;
+ int send_sigusr1 = 0;
int cont = 1;
int optchar;
int sock, con, rv, i;
@@ -483,7 +535,7 @@ int main(int argc, char *argv[])
int victim_host_id;
while (cont) {
- optchar = getopt(argc, argv, "Dp:i:hVws");
+ optchar = getopt(argc, argv, "Dp:i:hVws1");
switch (optchar) {
case 'D':
@@ -506,6 +558,9 @@ int main(int argc, char *argv[])
case 's':
send_opts = 1;
break;
+ case '1':
+ send_sigusr1 = 1;
+ break;
case 'h':
print_usage();
exit(0);
@@ -522,6 +577,11 @@ int main(int argc, char *argv[])
};
}
+ if (send_sigusr1) {
+ rv = send_signal(SIGUSR1);
+ return rv;
+ }
+
if (wait_opts && send_opts) {
fprintf(stderr, "-w and -s options cannot be used together\n");
exit(1);
@@ -584,6 +644,12 @@ int main(int argc, char *argv[])
goto out_refcount;
}
+ rv = sanlock_killpath(sock, 0, "fence_sanlockd", (char *)"-1");
+ if (rv < 0) {
+ log_error("killpath error %d", sock);
+ goto out_refcount;
+ }
+
rv = sanlock_restrict(sock, SANLK_RESTRICT_SIGKILL);
if (rv < 0) {
log_error("restrict error %d", sock);
diff --git a/src/cmd.c b/src/cmd.c
index 19416be..8ad72b7 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -310,6 +310,8 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
token->io_timeout = spi.io_timeout;
if (cl->restricted & SANLK_RESTRICT_SIGKILL)
token->flags |= T_RESTRICT_SIGKILL;
+ if (cl->restricted & SANLK_RESTRICT_SIGTERM)
+ token->flags |= T_RESTRICT_SIGTERM;
/* save a record of what this token_id is for later debugging */
log_level(spi.space_id, token->token_id, NULL, LOG_WARNING,
diff --git a/src/helper.c b/src/helper.c
index 2fce221..27898d5 100644
--- a/src/helper.c
+++ b/src/helper.c
@@ -103,7 +103,7 @@ static void run_path(struct helper_msg *hm)
av[av_count++] = strdup(arg);
}
- execv(av[0], av);
+ execvp(av[0], av);
}
static int read_hm(int fd, struct helper_msg *hm)
diff --git a/src/resource.c b/src/resource.c
index 2a60a16..226a6ce 100644
--- a/src/resource.c
+++ b/src/resource.c
@@ -564,6 +564,8 @@ static struct resource *new_resource(struct token *token)
r->pid = token->pid;
if (token->flags & T_RESTRICT_SIGKILL)
r->flags |= R_RESTRICT_SIGKILL;
+ if (token->flags & T_RESTRICT_SIGTERM)
+ r->flags |= R_RESTRICT_SIGTERM;
}
return r;
@@ -856,27 +858,20 @@ static void do_request(struct token *tt, int pid, uint32_t force_mode)
memset(&hm, 0, sizeof(hm));
- if (force_mode == SANLK_REQ_KILL_PID) {
+ if (force_mode == SANLK_REQ_FORCE) {
hm.type = HELPER_MSG_KILLPID;
hm.pid = pid;
hm.sig = (flags & R_RESTRICT_SIGKILL) ? SIGTERM : SIGKILL;
- } else if (force_mode == SANLK_REQ_SIGUSR1) {
- hm.type = HELPER_MSG_KILLPID;
- hm.pid = pid;
- hm.sig = SIGUSR1;
- } else if (force_mode == SANLK_REQ_KILLPATH) {
- if (!killpath[0]) {
- log_error("do_request %d force_mode %d no killpath",
- pid, force_mode);
- return;
+ } else if (force_mode == SANLK_REQ_GRACEFUL) {
+ if (killpath[0]) {
+ hm.type = HELPER_MSG_RUNPATH;
+ memcpy(hm.path, killpath, SANLK_HELPER_PATH_LEN);
+ memcpy(hm.args, killargs, SANLK_HELPER_ARGS_LEN);
+ } else {
+ hm.type = HELPER_MSG_KILLPID;
+ hm.pid = pid;
+ hm.sig = (flags & R_RESTRICT_SIGTERM) ? SIGKILL : SIGTERM;
}
- /* there's no thread locking for the clients array so
- we can't go searching for killpath/killargs for this
- pid from here, so we copy the info into struct
- resource so we can use it here. */
- hm.type = HELPER_MSG_RUNPATH;
- memcpy(hm.path, killpath, SANLK_HELPER_PATH_LEN);
- memcpy(hm.args, killargs, SANLK_HELPER_ARGS_LEN);
} else {
log_error("do_request %d unknown force_mode %d",
pid, force_mode);
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index 34ce260..f0153c6 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -68,7 +68,8 @@ struct sync_disk {
*/
#define T_RESTRICT_SIGKILL 0x00000001 /* inherited from client->restricted */
-#define T_LS_DEAD 0x00000002 /* don't bother trying to release if ls is dead */
+#define T_RESTRICT_SIGTERM 0x00000002 /* inherited from client->restricted */
+#define T_LS_DEAD 0x00000004 /* don't bother trying to release if ls is dead */
struct token {
/* values copied from acquire res arg */
@@ -99,6 +100,7 @@ struct token {
#define R_THREAD_EXAMINE 0x00000002
#define R_THREAD_RELEASE 0x00000004
#define R_RESTRICT_SIGKILL 0x00000008 /* inherited from token */
+#define R_RESTRICT_SIGTERM 0x00000010 /* inherited from token */
struct resource {
struct list_head list;
diff --git a/src/sanlock_resource.h b/src/sanlock_resource.h
index f68fc3e..3148b25 100644
--- a/src/sanlock_resource.h
+++ b/src/sanlock_resource.h
@@ -33,15 +33,21 @@
/*
* request force_mode
- * SANLK_REQ_KILL_PID: send SIGKILL to pid holding the resource, or
- * SIGTERM if SIGKILL is restricted
- * SANLK_REQ_SIGUSR1: send SIGUSR1 to pid holding the resource
- * SANLK_REQ_KILLPATH: run configured killpath against the pid
+ *
+ * SANLK_REQ_FORCE (SANLK_REQ_KILL_PID deprecated)
+ * Send SIGKILL to the pid holding the resource
+ * (or SIGTERM if SIGKILL is restricted.)
+ *
+ * SANLK_REQ_GRACEFUL
+ * Run killpath against the pid if it is defined, otherwise
+ * send SIGTERM to the pid (or SIGKILL if SIGTERM is restricted).
*/
-#define SANLK_REQ_KILL_PID 0x00000001
-#define SANLK_REQ_SIGUSR1 0x00000002
-#define SANLK_REQ_KILLPATH 0x00000003
+#define SANLK_REQ_FORCE 0x00000001
+#define SANLK_REQ_GRACEFUL 0x00000002
+
+/* old name deprecated */
+#define SANLK_REQ_KILL_PID SANLK_REQ_FORCE
int sanlock_register(void);
