fence_sanlock/fence_sanlock.in | 147 +++++++++++++++++++++++++----------------
fence_sanlock/fence_sanlockd.c | 22 +++---
2 files changed, 104 insertions(+), 65 deletions(-)
New commits:
commit e467b2092ade617fa0ed107a614fdac0259c0ec5
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Sep 18 13:47:00 2012 -0500
fence_sanlock: various fixes and changes
- change device arg name to path
- add some metadata descriptions
- in off, make request after first acquire fails
- in off, always success after acquire works
- in off, check if victim re-acquired cleanly
- in off, check if already off at start
- in status, use ^timestamp
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/fence_sanlock/fence_sanlock.in b/fence_sanlock/fence_sanlock.in
index 1cea274..1140880 100755
--- a/fence_sanlock/fence_sanlock.in
+++ b/fence_sanlock/fence_sanlock.in
@@ -30,10 +30,10 @@
# </unfence>
# </clusternode>
#
-# <fencedevice name="wd" agent="fence_sanlock"
device="/dev/fence_sanlock/leases"/>
+# <fencedevice name="wd" agent="fence_sanlock"
path="/dev/fence_sanlock/leases"/>
# setup (one time only):
-# fence_sanlock sanlock_init $device
+# fence_sanlock -o sanlock_init -p $path
#
# startup:
# service wdmd start
@@ -44,7 +44,7 @@
max_hosts=128
opts=
action=
-device=
+path=
host_id=
offset=
@@ -56,14 +56,14 @@ help() {
echo "Options:"
echo " -o <action> Action: off (default), on, status or metadata"
echo " (sanlock specific actions: sanlock_init)"
- echo " -d <path> sanlock shared storage for leases"
+ echo " -p <path> sanlock shared storage for leases"
echo " -i <num> sanlock host_id of node to operate on"
echo " -h Print this help, then exit"
echo " -V Print program version information, then exit"
echo ""
echo "stdin options:"
echo " action=<action>"
- echo " device=<path>"
+ echo " path=<path>"
echo " host_id=<num>"
}
@@ -74,8 +74,8 @@ cli_options() {
action=$2
shift
;;
- -d)
- device=$2
+ -p)
+ path=$2
shift
;;
-i)
@@ -103,8 +103,8 @@ stdin_options() {
action)
action=$val
;;
- device)
- device=$val
+ path)
+ path=$val
;;
host_id)
host_id=$val
@@ -117,7 +117,7 @@ stdin_options() {
if [ $# -eq 0 ]; then
stdin_options
else
- opts=$(getopt n:o:d:i:hV $@)
+ opts=$(getopt n:o:p:i:hV $@)
if [ "$?" != 0 ]; then
help
exit 1
@@ -125,13 +125,15 @@ else
cli_options $opts
fi
-# FIXME: Dave to review descriptions
metadata() {
cat << EOF
<?xml version="1.0" ?>
-<resource-agent name="fence_sanlock" shortdesc="Fence agent for sanlock
shared storage" >
+<resource-agent name="fence_sanlock" shortdesc="Fence agent for
watchdog and shared storage">
<longdesc>
-fence_sanlock is ... add long description here.
+fence_sanlock is an i/o fencing agent that uses the watchdog device to
+reset nodes. Shared storage (block or file) is used by sanlock to ensure
+that fenced nodes are reset, and to notify partitioned nodes that they
+need to be reset.
</longdesc>
<
vendor-url>http://www.redhat.com/</vendor-url>
<parameters>
@@ -140,15 +142,15 @@ fence_sanlock is ... add long description here.
<content type="string" default="off" />
<shortdesc lang="en">Fencing Action</shortdesc>
</parameter>
- <parameter name="device" unique="0" required="1">
- <getopt mixed="-d <action>" />
+ <parameter name="path" unique="0" required="1">
+ <getopt mixed="-p <action>" />
<content type="string" />
- <shortdesc lang="en">Path to sanlock shared storage
device</shortdesc>
+ <shortdesc lang="en">Path to sanlock shared storage</shortdesc>
</parameter>
<parameter name="host_id" unique="0" required="1">
<getopt mixed="-i <action>" />
<content type="string" />
- <shortdesc lang="en">Path to sanlock shared storage
device</shortdesc>
+ <shortdesc lang="en">Host id for sanlock (1-128)</shortdesc>
</parameter>
</parameters>
<actions>
@@ -163,31 +165,31 @@ EOF
return 0
}
-verify_device() {
- # verify device has been initialized
+verify_path() {
+ # verify storage has been initialized
- leader=$(sanlock direct read_leader -r fence:h$host_id:$device:$offset 2>&1)
+ leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1)
[ "$?" != 0 ] && {
- echo "Unable to read $device information"
+ echo "Unable to read $path"
return 1
}
magic="$(echo "$leader" | grep magic | awk '{print $NF}')"
[ -z "$magic" ] && {
- echo "Unable to determine $device sanlock magic"
+ echo "Unable to determine $path sanlock magic"
return 1
}
[ "$magic" != "0x6152010" ] && {
- echo "Error: $device magic does not match sanlock magic"
+ echo "Error: $path magic $magic does not match sanlock magic 0x6152010"
return 1
}
return 0
}
action_on() {
- verify_device || return 1
+ verify_path || return 1
[ -z "$(pidof fence_sanlockd)" ] && {
- daemonerr="$(fence_sanlockd -d $device -i $host_id 2>&1)"
+ daemonerr="$(fence_sanlockd -p $path -i $host_id 2>&1)"
[ "$?" != 0 ] && {
echo "Unable to execute fence_sanlockd. Error:"
echo "$daemonerr"
@@ -199,7 +201,7 @@ action_on() {
# it can take minutes, and we can't allow fence_tool join
# until this is complete
- while ! sanlock client status | grep -q fence:h$host_id:$device:$offset; do
+ while ! sanlock client status | grep -q fence:h$host_id:$path:$offset; do
# FIXME: check that r is really done being acquired?
# just appearing in output may not be enough
@@ -210,60 +212,97 @@ action_on() {
}
action_off() {
- verify_device || return 1
+ verify_path || return 1
owner_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
owner_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
ver="$(echo "$leader" | grep lver | awk '{print $NF}')"
+ timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
- # owner_id should equal host_id
- [ "$owner_id" != "$host_id" ] && {
- echo "Error: owner_id $owner_id does not match host_id $host_id"
+ [ -z "$timestamp" ] && {
+ echo "Unable to determine timestamp"
return 1
}
- sanlock client request -r fence:h$host_id:$device:$offset:$((ver + 1)) -f 2 >
/dev/null 2>&1
- [ "$?" != 0 ] && {
- echo "Unable to send sanlock client request"
+ # lease is released, so host is off
+ [ "$timestamp" = 0 ] && {
+ return 0
+ }
+
+ # owner_id should equal host_id
+ [ "$owner_id" != "$host_id" ] && {
+ echo "victim lease $host_id owned by $owner_id:$owner_gen"
return 1
}
pid="$(pidof fence_sanlockd)"
[ -z "$pid" ] && {
- echo "Unable to determine fence_sanlockd pin"
+ echo "Unable to determine fence_sanlockd pid"
return 1
}
+ loop=0
+
# FIXME: should this loop have a retry limit?
while :
do
- sanlock client acquire -r fence:h$host_id:$device:$offset -p $pid
+ loop=$(($loop+1))
+
+ sanlock client acquire -r fence:h$host_id:$path:$offset -p $pid
[ "$?" = 0 ] && {
# fence success
- sanlock client release -r fence:h$host_id:$device:$offset -p $pid
+ sanlock client release -r fence:h$host_id:$path:$offset -p $pid
[ "$?" != 0 ] && {
- # FIXME: what should we do here?
- echo "Unable to release lock?"
- #return 1
+ echo "release $host_id error $?"
}
- break;
+ return 0
}
- sleep 5
+ if [ "$loop" = 1 ]; then
+
+ # acquire probably failed because the victim is
+ # still alive and renewing its lease, (we could
+ # verify that by checking the error code, but the
+ # error codes are currently messed up due to
+ # negation). use a request on the victim's lease
+ # to tell it that it's being fenced and needs to
+ # reset. the -f 2 causes SIGUSR1 to be sent to
+ # fence_sanlockd on the victim.
+
+ sanlock client request -r fence:h$host_id:$path:$offset:$((ver + 1)) -f 2 >
/dev/null 2>&1
+ [ "$?" != 0 ] && {
+ echo "request $host_id error $?"
+ }
+ fi
+
+ sleep 10
+
+ # Reread the leader; if the victim's lease has been
+ # reacquired cleanly by the victim host (same host_id, new
+ # generation), we can quit with success
- # FIXME: there probably some errors here where we should just fail
+ leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1)
+ tmp_id="$(echo "$leader" | grep owner_id | awk '{print
$NF}')"
+ tmp_gen="$(echo "$leader" | grep owner_gen | awk '{print
$NF}')"
- # FIXME: we should probably reread the leader, and if it's been
- # reacquired cleanly by the host, I think we can quit with success
+ if [ "$owner_id" -eq "$tmp_id" ] && [
"$owner_gen" -lt "$tmp_gen" ]; then
+ echo "victim $owner_id:$owner_gen reacquired lease gen $tmp_gen"
+ return 0
+ fi
+
+ if [ "$owner_id" -ne "$tmp_id" ]; then
+ echo "victim $owner_id:$owner_gen acquired by $tmp_id:$tmp_gen"
+ return 1
+ fi
done
return 0
}
action_status() {
- verify_device || return 1
+ verify_path || return 1
- timestamp="$(echo "$leader" | grep timestamp | awk '{print
$NF}')"
+ timestamp="$(echo "$leader" | grep ^timestamp | awk '{print
$NF}')"
[ -z "$timestamp" ] && {
echo "Unable to determine timestamp"
@@ -282,17 +321,17 @@ action_status() {
}
sanlock_init() {
- # initialize lease device
- echo -n "Initializing lease device $device: "
- sanlock direct init -s fence:0:$device:0 \
+ # initialize lease path
+ echo -n "Initializing fence sanlock lockspace on $path: "
+ sanlock direct init -s fence:0:$path:0 \
/dev/null 2>/dev/null || \
{ echo "error $?"
&& return 1; }
echo "ok"
- echo -n "Initializing host leases: "
+ echo -n "Initializing $max_hosts sanlock host leases on $path: "
for host_id in $(seq 1 $max_hosts); do
offset=$((host_id * 1048576))
- sanlock direct init -r fence:h$host_id:$device:$offset \
+ sanlock direct init -r fence:h$host_id:$path:$offset \
/dev/null 2>/dev/null || \
{ echo "error $? for
host $host_id" && return 1; }
done
@@ -304,10 +343,10 @@ sanlock_init() {
[ -z "$action" ] && action=off
# check actions and options compatibility
-# all actions beside metadata needs device
+# all actions beside metadata needs storage
[ "$action" != "metadata" ] && {
- [ -z "$device" ] && {
- echo "device argument required"
+ [ -z "$path" ] && {
+ echo "storage path argument required"
exit 1
}
# all actions beside sanlock_init needs host_id
diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c
index 6a0de61..ea7ffdc 100644
--- a/fence_sanlock/fence_sanlockd.c
+++ b/fence_sanlock/fence_sanlockd.c
@@ -42,7 +42,7 @@
* start before cman and stop after cman. In future
* we could have the init script start the daemon,
* but that would also require a new config scheme
- * since we wouldn't be getting the device and host_id
+ * since we wouldn't be getting the path and host_id
* from cluster.conf via fence_node.
*
* simplest would be to have cman init just send us
@@ -64,7 +64,7 @@ static int shutdown;
static int we_are_victim;
static int daemon_debug;
static int our_host_id;
-static char device[PATH_MAX];
+static char path[PATH_MAX];
static struct sanlk_lockspace ls;
static struct sanlk_resource *r;
static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)];
@@ -260,11 +260,11 @@ static int setup_signals(void)
static void print_usage(void)
{
printf("Usage:\n");
- printf("fence_sanlockd -d <device> -i <host_id>\n");
+ printf("fence_sanlockd -p <path> -i <host_id>\n");
printf("\n");
printf("Options:\n");
printf(" -D Enable debugging to stderr and don't fork\n");
- printf(" -d <path> Path to shared storage with sanlock leases\n");
+ printf(" -p <path> Path to shared storage with sanlock leases\n");
printf(" -i <host_id> Local sanlock host_id\n");
printf(" -h Print this help, then exit\n");
printf(" -V Print program version information, then exit\n");
@@ -282,14 +282,14 @@ int main(int argc, char *argv[])
int sock, con, rv, i;
while (cont) {
- optchar = getopt(argc, argv, "Dd:i:hV");
+ optchar = getopt(argc, argv, "Dp:i:hV");
switch (optchar) {
case 'D':
daemon_debug = 1;
break;
- case 'd':
- strcpy(device, optarg);
+ case 'p':
+ strcpy(path, optarg);
break;
case 'i':
our_host_id = atoi(optarg);
@@ -316,9 +316,9 @@ int main(int argc, char *argv[])
exit(1);
}
- if (!device[0]) {
+ if (!path[0]) {
daemon_debug = 1;
- log_error("device arg required");
+ log_error("path arg required");
exit(1);
}
@@ -370,7 +370,7 @@ int main(int argc, char *argv[])
}
memset(&ls, 0, sizeof(ls));
- sprintf(ls.host_id_disk.path, "%s", device);
+ sprintf(ls.host_id_disk.path, "%s", path);
strcpy(ls.name, "fence");
ls.host_id = our_host_id;
@@ -388,7 +388,7 @@ int main(int argc, char *argv[])
r = (struct sanlk_resource *)&rdbuf;
strcpy(r->lockspace_name, "fence");
sprintf(r->name, "h%d", our_host_id);
- sprintf(r->disks[0].path, "%s", device);
+ sprintf(r->disks[0].path, "%s", path);
r->disks[0].offset = our_host_id * 1048576;
r->num_disks = 1;