fence_sanlock/fence_sanlock.in | 48 +++++-------
fence_sanlock/fence_sanlockd.c | 160 ++++++++++++++++++++++++++++++++++++-----
2 files changed, 167 insertions(+), 41 deletions(-)
New commits:
commit 401d1830a7c0ba31789ee35d25d71aa5dc3a8889
Author: David Teigland <teigland(a)redhat.com>
Date: Thu Sep 20 17:16:06 2012 -0500
fence_sanlockd: allow options to be delayed
When fence_sanlockd is started with -w, it will
wait for fence_sanlockd -s to send it the
-i and -p options.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index e491ded..7f62e0a 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -60,8 +60,9 @@
#define RUN_DIR "/var/run/fence_sanlockd"
static char *prog_name = (char *)"fence_sanlockd";
-static int shutdown;
+
static int we_are_victim;
+static int shutdown;
static int daemon_debug;
static int our_host_id;
static char path[PATH_MAX];
@@ -70,6 +71,13 @@ static struct sanlk_resource *r;
static struct sanlk_disk disk;
static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)];
static char lockfile_path[PATH_MAX];
+static char fifo_path[PATH_MAX];
+static char fifo_line[PATH_MAX];
+static char key1[PATH_MAX];
+static char key2[PATH_MAX];
+static char val1[PATH_MAX];
+static char val2[PATH_MAX];
+
struct client {
int used;
@@ -258,17 +266,105 @@ static int setup_signals(void)
return 0;
}
+static int wait_options(void)
+{
+ int fd, rv;
+
+ snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR);
+
+ rv = mkfifo(fifo_path, (S_IRUSR | S_IWUSR));
+ if (rv && errno != EEXIST) {
+ log_error("wait_options mkfifo error %d %s", errno, fifo_path);
+ return -1;
+ }
+
+ fd = open(fifo_path, O_RDONLY|O_CLOEXEC);
+ if (fd < 0) {
+ log_error("wait_options fifo open error %d %s", errno, fifo_path);
+ unlink(fifo_path);
+ return -1;
+ }
+
+ reread:
+ memset(fifo_line, 0, sizeof(fifo_line));
+
+ rv = read(fd, fifo_line, sizeof(fifo_line));
+ if (rv < 0) {
+ log_error("wait_options read error %d", errno);
+ goto out;
+ }
+
+ rv = sscanf(fifo_line, "%s %s %s %s", key1, val1, key2, val2);
+ if (rv != 4) {
+ log_error("wait_options scan error %d", rv);
+ goto reread;
+ }
+
+ if (strcmp(key1, "-p") || strcmp(key2, "-i")) {
+ log_error("wait_options args error");
+ goto reread;
+ }
+
+ strncpy(path, val1, PATH_MAX-1);
+ our_host_id = atoi(val2);
+
+ if (!our_host_id || our_host_id > MAX_HOSTS) {
+ log_error("wait_options invalid host_id");
+ goto reread;
+ }
+
+ if (!path[0]) {
+ log_error("wait_options invalid path");
+ goto reread;
+ }
+
+ log_debug("wait_options -p %s -i %d", path, our_host_id);
+ out:
+ close(fd);
+ unlink(fifo_path);
+ return 0;
+}
+
+static int send_options(void)
+{
+ int fd, rv;
+
+ snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR);
+
+ fd = open(fifo_path, O_WRONLY|O_CLOEXEC);
+ if (fd < 0) {
+ fprintf(stderr, "fifo open error %d %s\n", errno, fifo_path);
+ return -1;
+ }
+
+ memset(fifo_line, 0, sizeof(fifo_line));
+
+ snprintf(fifo_line, PATH_MAX-1, "-p %s -i %d", path, our_host_id);
+
+ rv = write(fd, fifo_line, sizeof(fifo_line));
+ if (rv < 0) {
+ fprintf(stderr, "fifo write error %d\n", errno);
+ } else {
+ rv = 0;
+ }
+
+ close(fd);
+ return rv;
+}
+
static void print_usage(void)
{
printf("Usage:\n");
- printf("fence_sanlockd -p <path> -i <host_id>\n");
+ printf("fence_sanlockd [options]\n");
printf("\n");
printf("Options:\n");
- printf(" -D Enable debugging to stderr and don't fork\n");
- printf(" -p <path> Path to shared storage with sanlock leases\n");
- printf(" -i <host_id> Local sanlock host_id\n");
- printf(" -h Print this help, then exit\n");
- printf(" -V Print program version information, then exit\n");
+ printf(" -D Enable debugging to stderr and don't fork\n");
+ printf(" -p <path> Path to shared storage with sanlock leases\n");
+ printf(" -i <host_id> Local sanlock host_id (1-%d)\n", MAX_HOSTS);
+ printf(" -w Wait for fence_sanlockd -s to send options (p,i)\n");
+ printf(" -s Send options (p,i) to waiting fence_sanlockd -w\n");
+ printf(" -h Print this help, then exit\n");
+ printf(" -V Print program version information, then exit\n");
}
int main(int argc, char *argv[])
@@ -278,13 +374,14 @@ int main(int argc, char *argv[])
uint64_t live_time, now;
int poll_timeout;
int sleep_seconds;
+ int send_opts = 0, wait_opts = 0;
int cont = 1;
int optchar;
int sock, con, rv, i;
int align;
while (cont) {
- optchar = getopt(argc, argv, "Dp:i:hV");
+ optchar = getopt(argc, argv, "Dp:i:hVws");
switch (optchar) {
case 'D':
@@ -295,6 +392,17 @@ int main(int argc, char *argv[])
break;
case 'i':
our_host_id = atoi(optarg);
+ if (our_host_id > MAX_HOSTS) {
+ fprintf(stderr, "invalid host_id %d, use 1-%d\n",
+ our_host_id, MAX_HOSTS);
+ exit(1);
+ }
+ break;
+ case 'w':
+ wait_opts = 1;
+ break;
+ case 's':
+ send_opts = 1;
break;
case 'h':
print_usage();
@@ -312,18 +420,21 @@ int main(int argc, char *argv[])
};
}
- if (!our_host_id || our_host_id > MAX_HOSTS) {
- daemon_debug = 1;
- log_error("-i %d invalid, our_host_id must be between 1 and %d", our_host_id,
MAX_HOSTS);
+ if (wait_opts && send_opts) {
+ fprintf(stderr, "-w and -s options cannot be used together\n");
exit(1);
}
- if (!path[0]) {
- daemon_debug = 1;
- log_error("path arg required");
+ if (!wait_opts && (!our_host_id || !path[0])) {
+ fprintf(stderr, "-i and -p options required\n");
exit(1);
}
+ if (send_opts) {
+ rv = send_options();
+ return rv;
+ }
+
if (!daemon_debug) {
if (daemon(0, 0) < 0) {
fprintf(stderr, "cannot fork daemon\n");
@@ -341,6 +452,12 @@ int main(int argc, char *argv[])
if (rv < 0)
goto out_lockfile;
+ if (wait_opts) {
+ rv = wait_options();
+ if (rv < 0)
+ goto out_lockfile;
+ }
+
con = wdmd_connect();
if (con < 0) {
log_error("wdmd connect error %d", con);
commit 0a80be08c4f85b9e1ab1c9db7911def51e213faf
Author: David Teigland <teigland(a)redhat.com>
Date: Thu Sep 20 14:06:00 2012 -0500
fence_sanlock: variable alignment
- make lease size depend on disk sector size
- clarify fence_sanlock read_leader behavior
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/fence_sanlock/fence_sanlock.in b/fence_sanlock/fence_sanlock.in
index a6f9178..f43acc2 100755
--- a/fence_sanlock/fence_sanlock.in
+++ b/fence_sanlock/fence_sanlock.in
@@ -165,35 +165,24 @@ EOF
return 0
}
-get_leader_info() {
+read_leader() {
+ # verify storage has been initialized
+
leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1)
[ "$?" != 0 ] && {
- echo "Unable to read $path"
+ echo "Error: unable to read $path"
return 1
}
- timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
-
- [ -z "$timestamp" ] && {
- echo "Unable to determine timestamp"
- return 1
- }
-
- return 0
-}
-
-verify_path() {
- # verify storage has been initialized
-
- get_leader_info || return 1
-
magic="$(echo "$leader" | grep magic | awk '{print $NF}')"
+
[ -z "$magic" ] && {
- echo "Unable to determine $path sanlock magic"
+ echo "Error: no sanlock magic number at $path:$offset"
return 1
}
+
[ "$magic" != "0x6152010" ] && {
- echo "Error: $path magic $magic does not match sanlock magic 0x6152010"
+ echo "Error: invalid sanlock magic number $magic at $path:$offset"
return 1
}
@@ -201,7 +190,7 @@ verify_path() {
}
action_on() {
- verify_path || return 1
+ read_leader || return 1
[ -z "$(pidof fence_sanlockd)" ] && {
daemonerr="$(fence_sanlockd -p $path -i $host_id 2>&1)"
@@ -245,11 +234,12 @@ action_on() {
}
action_off() {
- verify_path || return 1
+ read_leader || return 1
owner_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
owner_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
ver="$(echo "$leader" | grep lver | awk '{print $NF}')"
+ timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
# lease is released, so host is off
[ "$timestamp" = 0 ] && {
@@ -308,7 +298,7 @@ action_off() {
# reacquired cleanly by the victim host (same host_id, new
# generation), we can quit with success
- get_leader_info || return 1
+ read_leader || return 1
tmp_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
tmp_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
@@ -328,7 +318,9 @@ action_off() {
}
action_status() {
- verify_path || return 1
+ read_leader || return 1
+
+ timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
# lease is released, so host is "off"
[ "$timestamp" = 0 ] && {
@@ -351,7 +343,7 @@ sanlock_init() {
echo -n "Initializing $max_hosts sanlock host leases on $path: "
for host_id in $(seq 1 $max_hosts); do
- offset=$((host_id * 1048576))
+ offset=$((host_id * $align))
sanlock direct init -r fence:h$host_id:$path:$offset \
/dev/null 2>/dev/null || \
{ echo "error $? for
host $host_id" && return 1; }
@@ -365,16 +357,22 @@ sanlock_init() {
# check actions and options compatibility
# all actions beside metadata needs storage
+
[ "$action" != "metadata" ] && {
[ -z "$path" ] && {
echo "storage path argument required"
exit 1
}
+
# all actions beside sanlock_init needs host_id
[ "$action" != "sanlock_init" ] && [ -z "$host_id"
] && {
echo "host_id argument required"
exit 1
}
+
+ # FIXME: add direct align command to sanlock
+ # align="&(sanlock direct align $path)"
+ align=1048576
}
# verify host_id parameter
@@ -383,7 +381,7 @@ sanlock_init() {
echo "host_id must be between 1 and $max_hosts"
exit 1
fi
- offset=$((host_id * 1048576))
+ offset=$((host_id * $align))
}
case "$action" in
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index ea7ffdc..e491ded 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -28,11 +28,11 @@
#include "sanlock.h"
#include "sanlock_admin.h"
#include "sanlock_resource.h"
+#include "sanlock_direct.h"
#include "wdmd.h"
/*
* TODO:
- * handle 4k disk blocks using sanlock_align()
* variable i/o timeouts?
*
* shutdown: how/when does SIGTERM happen?
@@ -67,6 +67,7 @@ static int our_host_id;
static char path[PATH_MAX];
static struct sanlk_lockspace ls;
static struct sanlk_resource *r;
+static struct sanlk_disk disk;
static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)];
static char lockfile_path[PATH_MAX];
@@ -280,6 +281,7 @@ int main(int argc, char *argv[])
int cont = 1;
int optchar;
int sock, con, rv, i;
+ int align;
while (cont) {
optchar = getopt(argc, argv, "Dp:i:hV");
@@ -369,6 +371,15 @@ int main(int argc, char *argv[])
goto out_refcount;
}
+ memset(&disk, 0, sizeof(disk));
+ sprintf(disk.path, "%s", path);
+
+ align = sanlock_direct_align(&disk);
+ if (align < 0) {
+ log_error("direct_align error %d", align);
+ goto out_refcount;
+ }
+
memset(&ls, 0, sizeof(ls));
sprintf(ls.host_id_disk.path, "%s", path);
strcpy(ls.name, "fence");
@@ -389,7 +400,7 @@ int main(int argc, char *argv[])
strcpy(r->lockspace_name, "fence");
sprintf(r->name, "h%d", our_host_id);
sprintf(r->disks[0].path, "%s", path);
- r->disks[0].offset = our_host_id * 1048576;
+ r->disks[0].offset = our_host_id * align;
r->num_disks = 1;
log_debug("acquire begin");