Branch 'fence_sanlock' - 2 commits - fence_sanlock/fence_sanlock.in fence_sanlock/fence_sanlockd.c

fence_sanlock/fence_sanlock.in | 48 +++++------- fence_sanlock/fence_sanlockd.c | 160 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 167 insertions(+), 41 deletions(-) New commits: commit 401d1830a7c0ba31789ee35d25d71aa5dc3a8889 Author: David Teigland <teigland(a)redhat.com&gt; Date: Thu Sep 20 17:16:06 2012 -0500 fence_sanlockd: allow options to be delayed When fence_sanlockd is started with -w, it will wait for fence_sanlockd -s to send it the -i and -p options. Signed-off-by: David Teigland <teigland(a)redhat.com&gt; diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c index e491ded..7f62e0a 100644 --- a/fence_sanlock/fence_sanlockd.c +++ b/fence_sanlock/fence_sanlockd.c @@ -60,8 +60,9 @@ #define RUN_DIR "/var/run/fence_sanlockd" static char *prog_name = (char *)"fence_sanlockd"; -static int shutdown; + static int we_are_victim; +static int shutdown; static int daemon_debug; static int our_host_id; static char path[PATH_MAX]; @@ -70,6 +71,13 @@ static struct sanlk_resource *r; static struct sanlk_disk disk; static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; static char lockfile_path[PATH_MAX]; +static char fifo_path[PATH_MAX]; +static char fifo_line[PATH_MAX]; +static char key1[PATH_MAX]; +static char key2[PATH_MAX]; +static char val1[PATH_MAX]; +static char val2[PATH_MAX]; + struct client { int used; @@ -258,17 +266,105 @@ static int setup_signals(void) return 0; } +static int wait_options(void) +{ + int fd, rv; + + snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR); + + rv = mkfifo(fifo_path, (S_IRUSR | S_IWUSR)); + if (rv && errno != EEXIST) { + log_error("wait_options mkfifo error %d %s", errno, fifo_path); + return -1; + } + + fd = open(fifo_path, O_RDONLY|O_CLOEXEC); + if (fd < 0) { + log_error("wait_options fifo open error %d %s", errno, fifo_path); + unlink(fifo_path); + return -1; + } + + reread: + memset(fifo_line, 0, sizeof(fifo_line)); + + rv = read(fd, fifo_line, sizeof(fifo_line)); + if (rv < 0) { + log_error("wait_options read error %d", errno); + goto out; + } + + rv = sscanf(fifo_line, "%s %s %s %s", key1, val1, key2, val2); + if (rv != 4) { + log_error("wait_options scan error %d", rv); + goto reread; + } + + if (strcmp(key1, "-p") || strcmp(key2, "-i")) { + log_error("wait_options args error"); + goto reread; + } + + strncpy(path, val1, PATH_MAX-1); + our_host_id = atoi(val2); + + if (!our_host_id || our_host_id > MAX_HOSTS) { + log_error("wait_options invalid host_id"); + goto reread; + } + + if (!path[0]) { + log_error("wait_options invalid path"); + goto reread; + } + + log_debug("wait_options -p %s -i %d", path, our_host_id); + out: + close(fd); + unlink(fifo_path); + return 0; +} + +static int send_options(void) +{ + int fd, rv; + + snprintf(fifo_path, PATH_MAX-1, "%s/prog_name.fifo", RUN_DIR); + + fd = open(fifo_path, O_WRONLY|O_CLOEXEC); + if (fd < 0) { + fprintf(stderr, "fifo open error %d %s\n", errno, fifo_path); + return -1; + } + + memset(fifo_line, 0, sizeof(fifo_line)); + + snprintf(fifo_line, PATH_MAX-1, "-p %s -i %d", path, our_host_id); + + rv = write(fd, fifo_line, sizeof(fifo_line)); + if (rv < 0) { + fprintf(stderr, "fifo write error %d\n", errno); + } else { + rv = 0; + } + + close(fd); + return rv; +} + static void print_usage(void) { printf("Usage:\n"); - printf("fence_sanlockd -p <path> -i <host_id>\n"); + printf("fence_sanlockd [options]\n"); printf("\n"); printf("Options:\n"); - printf(" -D Enable debugging to stderr and don't fork\n"); - printf(" -p <path> Path to shared storage with sanlock leases\n"); - printf(" -i <host_id> Local sanlock host_id\n"); - printf(" -h Print this help, then exit\n"); - printf(" -V Print program version information, then exit\n"); + printf(" -D Enable debugging to stderr and don't fork\n"); + printf(" -p <path> Path to shared storage with sanlock leases\n"); + printf(" -i <host_id> Local sanlock host_id (1-%d)\n", MAX_HOSTS); + printf(" -w Wait for fence_sanlockd -s to send options (p,i)\n"); + printf(" -s Send options (p,i) to waiting fence_sanlockd -w\n"); + printf(" -h Print this help, then exit\n"); + printf(" -V Print program version information, then exit\n"); } int main(int argc, char *argv[]) @@ -278,13 +374,14 @@ int main(int argc, char *argv[]) uint64_t live_time, now; int poll_timeout; int sleep_seconds; + int send_opts = 0, wait_opts = 0; int cont = 1; int optchar; int sock, con, rv, i; int align; while (cont) { - optchar = getopt(argc, argv, "Dp:i:hV"); + optchar = getopt(argc, argv, "Dp:i:hVws"); switch (optchar) { case 'D': @@ -295,6 +392,17 @@ int main(int argc, char *argv[]) break; case 'i': our_host_id = atoi(optarg); + if (our_host_id > MAX_HOSTS) { + fprintf(stderr, "invalid host_id %d, use 1-%d\n", + our_host_id, MAX_HOSTS); + exit(1); + } + break; + case 'w': + wait_opts = 1; + break; + case 's': + send_opts = 1; break; case 'h': print_usage(); @@ -312,18 +420,21 @@ int main(int argc, char *argv[]) }; } - if (!our_host_id || our_host_id > MAX_HOSTS) { - daemon_debug = 1; - log_error("-i %d invalid, our_host_id must be between 1 and %d", our_host_id, MAX_HOSTS); + if (wait_opts && send_opts) { + fprintf(stderr, "-w and -s options cannot be used together\n"); exit(1); } - if (!path[0]) { - daemon_debug = 1; - log_error("path arg required"); + if (!wait_opts && (!our_host_id || !path[0])) { + fprintf(stderr, "-i and -p options required\n"); exit(1); } + if (send_opts) { + rv = send_options(); + return rv; + } + if (!daemon_debug) { if (daemon(0, 0) < 0) { fprintf(stderr, "cannot fork daemon\n"); @@ -341,6 +452,12 @@ int main(int argc, char *argv[]) if (rv < 0) goto out_lockfile; + if (wait_opts) { + rv = wait_options(); + if (rv < 0) + goto out_lockfile; + } + con = wdmd_connect(); if (con < 0) { log_error("wdmd connect error %d", con); commit 0a80be08c4f85b9e1ab1c9db7911def51e213faf Author: David Teigland <teigland(a)redhat.com&gt; Date: Thu Sep 20 14:06:00 2012 -0500 fence_sanlock: variable alignment - make lease size depend on disk sector size - clarify fence_sanlock read_leader behavior Signed-off-by: David Teigland <teigland(a)redhat.com&gt; diff --git a/fence_sanlock/fence_sanlock.in b/fence_sanlock/fence_sanlock.in index a6f9178..f43acc2 100755 --- a/fence_sanlock/fence_sanlock.in +++ b/fence_sanlock/fence_sanlock.in @@ -165,35 +165,24 @@ EOF return 0 } -get_leader_info() { +read_leader() { + # verify storage has been initialized + leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1) [ "$?" != 0 ] && { - echo "Unable to read $path" + echo "Error: unable to read $path" return 1 } - timestamp="$(echo "$leader" | grep ^timestamp | awk '{print $NF}')" - - [ -z "$timestamp" ] && { - echo "Unable to determine timestamp" - return 1 - } - - return 0 -} - -verify_path() { - # verify storage has been initialized - - get_leader_info || return 1 - magic="$(echo "$leader" | grep magic | awk '{print $NF}')" + [ -z "$magic" ] && { - echo "Unable to determine $path sanlock magic" + echo "Error: no sanlock magic number at $path:$offset" return 1 } + [ "$magic" != "0x6152010" ] && { - echo "Error: $path magic $magic does not match sanlock magic 0x6152010" + echo "Error: invalid sanlock magic number $magic at $path:$offset" return 1 } @@ -201,7 +190,7 @@ verify_path() { } action_on() { - verify_path || return 1 + read_leader || return 1 [ -z "$(pidof fence_sanlockd)" ] && { daemonerr="$(fence_sanlockd -p $path -i $host_id 2>&1)" @@ -245,11 +234,12 @@ action_on() { } action_off() { - verify_path || return 1 + read_leader || return 1 owner_id="$(echo "$leader" | grep owner_id | awk '{print $NF}')" owner_gen="$(echo "$leader" | grep owner_gen | awk '{print $NF}')" ver="$(echo "$leader" | grep lver | awk '{print $NF}')" + timestamp="$(echo "$leader" | grep ^timestamp | awk '{print $NF}')" # lease is released, so host is off [ "$timestamp" = 0 ] && { @@ -308,7 +298,7 @@ action_off() { # reacquired cleanly by the victim host (same host_id, new # generation), we can quit with success - get_leader_info || return 1 + read_leader || return 1 tmp_id="$(echo "$leader" | grep owner_id | awk '{print $NF}')" tmp_gen="$(echo "$leader" | grep owner_gen | awk '{print $NF}')" @@ -328,7 +318,9 @@ action_off() { } action_status() { - verify_path || return 1 + read_leader || return 1 + + timestamp="$(echo "$leader" | grep ^timestamp | awk '{print $NF}')" # lease is released, so host is "off" [ "$timestamp" = 0 ] && { @@ -351,7 +343,7 @@ sanlock_init() { echo -n "Initializing $max_hosts sanlock host leases on $path: " for host_id in $(seq 1 $max_hosts); do - offset=$((host_id * 1048576)) + offset=$((host_id * $align)) sanlock direct init -r fence:h$host_id:$path:$offset \ { echo "error $? for host $host_id" && return 1; } @@ -365,16 +357,22 @@ sanlock_init() { # check actions and options compatibility # all actions beside metadata needs storage + [ "$action" != "metadata" ] && { [ -z "$path" ] && { echo "storage path argument required" exit 1 } + # all actions beside sanlock_init needs host_id [ "$action" != "sanlock_init" ] && [ -z "$host_id" ] && { echo "host_id argument required" exit 1 } + + # FIXME: add direct align command to sanlock + # align="&(sanlock direct align $path)" + align=1048576 } # verify host_id parameter @@ -383,7 +381,7 @@ sanlock_init() { echo "host_id must be between 1 and $max_hosts" exit 1 fi - offset=$((host_id * 1048576)) + offset=$((host_id * $align)) } case "$action" in diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c index ea7ffdc..e491ded 100644 --- a/fence_sanlock/fence_sanlockd.c +++ b/fence_sanlock/fence_sanlockd.c @@ -28,11 +28,11 @@ #include "sanlock.h" #include "sanlock_admin.h" #include "sanlock_resource.h" +#include "sanlock_direct.h" #include "wdmd.h" /* * TODO: - * handle 4k disk blocks using sanlock_align() * variable i/o timeouts? * * shutdown: how/when does SIGTERM happen? @@ -67,6 +67,7 @@ static int our_host_id; static char path[PATH_MAX]; static struct sanlk_lockspace ls; static struct sanlk_resource *r; +static struct sanlk_disk disk; static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; static char lockfile_path[PATH_MAX]; @@ -280,6 +281,7 @@ int main(int argc, char *argv[]) int cont = 1; int optchar; int sock, con, rv, i; + int align; while (cont) { optchar = getopt(argc, argv, "Dp:i:hV"); @@ -369,6 +371,15 @@ int main(int argc, char *argv[]) goto out_refcount; } + memset(&disk, 0, sizeof(disk)); + sprintf(disk.path, "%s", path); + + align = sanlock_direct_align(&disk); + if (align < 0) { + log_error("direct_align error %d", align); + goto out_refcount; + } + memset(&ls, 0, sizeof(ls)); sprintf(ls.host_id_disk.path, "%s", path); strcpy(ls.name, "fence"); @@ -389,7 +400,7 @@ int main(int argc, char *argv[]) strcpy(r->lockspace_name, "fence"); sprintf(r->name, "h%d", our_host_id); sprintf(r->disks[0].path, "%s", path); - r->disks[0].offset = our_host_id * 1048576; + r->disks[0].offset = our_host_id * align; r->num_disks = 1; log_debug("acquire begin");