Getting started with sanlock/Ubuntu 12.04
by NEVEU Stephane
Hi all,
New to Sanlock, I try to make it work on Ubuntu 12.04 (using the following ppa : https://launchpad.net/~wb-munzinger/+archive/ppa-sanlock)
Libvirt-bin and libvirt-sanlock are both version 0.9.11-2.2.ubuntu.1
Sanlock is version 2.1-4.ubuntu.1
I just cannot make it work :
Here is my conf :
My NFS storage is mounted in fstab on both hosts :
MYIP:/SANLOCK /var/lib/libvirt/sanlock nfs hard,nointr,_netdev 0 0
First I don't want to reboot my hosts in case of concurrent access, I just don't want libvirt to be able to launch my qcow2, so as far as I understood, no need to modprode softdog (tell me if I'm wrong).
Cat /etc/default/sanlock
Enabled=1
Sanlock_opts="-w 0"
Must I use wdmd ? It's enabled=0 for now...
In my /etc/libvirt/qemu-sanlock.conf
Auto_disk_lease = 1
Disk_lease_dir = "/var/lib/libvirt/sanlock"
And I do have different host_id (1 and 2) !
In qemu.conf :
Lock_manager = "sanlock"
So I've got two .qcow2 on /var/lib/libvirt/sanlock and a __LIBVIRT__DISKS__:0 created :
Sanlock status :
P -1 listener
P -1 status
S __LIBVIRT__DISKS__:1:/var/lib/libvirt/sanlock/__LIBVIRT_DISKS__:0
(And S __LIBVIRT__DISKS__:2:/var/lib/libvirt/sanlock/__LIBVIRT_DISKS__:0 on the second host)
Now when I virsh start myvm1 on my 1st kvm, then a sanlock direct dump a __LIBVIRT__DISKS__ shows nothing except offset lockspace etc but no value (on both hosts).
I can start the same vm1 on the second KVM without warning/error (!)
Nevertheless, it seems that a lock is created on disks because a lock file is created (md5sum I guess).
When now I stop my VM, this md5 lock file is never removed. I must virt-sanlock-cleanup to remove it...
If you could help me, that would be really great.
Thank you.
10 years, 4 months
- 3
- 4
2 commits - src/cmd.c src/main.c src/sanlock.h src/sanlock_internal.h tests/sanlk_load.c
by David Teigland
src/cmd.c | 62 ++++++++++++++++++++++++-----
src/main.c | 10 +++-
src/sanlock.h | 3 -
src/sanlock_internal.h | 3 -
tests/sanlk_load.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 166 insertions(+), 16 deletions(-)
New commits:
commit 448abc7340e74793a566d3652d76c55f3dbcfa0a
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Nov 5 16:14:24 2013 -0600
tests: sanlk_load
Add "all" to test acquiring many resources.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/tests/sanlk_load.c b/tests/sanlk_load.c
index 8e87ab7..0a6d39f 100644
--- a/tests/sanlk_load.c
+++ b/tests/sanlk_load.c
@@ -602,6 +602,45 @@ int do_rand_child(void)
return 0;
}
+int do_all_child(void)
+{
+ int sx, rx, full;
+ int fd, rv;
+ int pid = getpid();
+
+ srandom(pid);
+
+ memset(lock_state, 0, sizeof(lock_state));
+
+ fd = sanlock_register();
+ if (fd < 0) {
+ log_error("%d sanlock_register error %d", pid, fd);
+ exit(-1);
+ }
+
+ while (!prog_stop) {
+ for (sx = 0; sx < ls_count; sx++) {
+ for (rx = 0; rx < res_count; rx++) {
+ rv = acquire_one(pid, fd, sx, rx, EX, &full);
+ if (!rv)
+ lock_state[sx][rx] = EX;
+ }
+
+ inquire_all(pid, fd);
+
+ for (rx = 0; rx < res_count-1; rx++) {
+ rv = release_one(pid, fd, sx, rx);
+ lock_state[sx][rx] = UN;
+ }
+
+ inquire_all(pid, fd);
+ }
+ }
+
+ display_rv(pid);
+ return 0;
+}
+
/*
* sanlk_load rand <lock_disk_base> -i <host_id> [-D -s <ls_count> -r <res_count> -p <pid_count>]
*/
@@ -822,6 +861,68 @@ int do_rand(int argc, char *argv[])
return 0;
}
+int do_all(int argc, char *argv[])
+{
+ struct sigaction act;
+ int children[MAX_PID_COUNT];
+ int run_count = 0;
+ int i, rv, pid, status;
+
+ if (argc < 5)
+ return -1;
+
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = sigterm_handler;
+ sigaction(SIGTERM, &act, NULL);
+
+ strcpy(lock_disk_base, argv[2]);
+
+ get_options(argc, argv);
+
+ rv = add_lockspaces();
+ if (rv < 0)
+ return rv;
+
+ printf("forking %d pids\n", pid_count);
+
+ for (i = 0; i < pid_count; i++) {
+ pid = fork();
+
+ if (pid < 0) {
+ log_error("fork %d failed %d run_count %d", i, errno, run_count);
+ break;
+ }
+ if (!pid) {
+ do_all_child();
+ exit(-1);
+ }
+ children[i] = pid;
+ run_count++;
+ }
+
+ printf("children running\n");
+
+ while (!prog_stop) {
+ sleep(1);
+ }
+
+ printf("stopping pids");
+
+ for (i = 0; i < pid_count; i++)
+ kill(children[i], SIGTERM);
+
+ while (run_count) {
+ pid = wait(&status);
+ if (pid > 0) {
+ run_count--;
+ printf(".");
+ }
+ }
+ printf("\n");
+
+ return 0;
+}
+
/*
* sanlk_load init <lock_disk_base> [<ls_count> <res_count>]
* lock_disk_base = /dev/vg/foo
@@ -903,6 +1004,9 @@ int main(int argc, char *argv[])
else if (!strcmp(argv[1], "rand"))
rv = do_rand(argc, argv);
+ else if (!strcmp(argv[1], "all"))
+ rv = do_all(argc, argv);
+
if (!rv)
return 0;
commit 8aeada00fbf899756f8710f02d74fb1ef969d635
Author: David Teigland <teigland(a)redhat.com>
Date: Tue Nov 5 16:10:00 2013 -0600
sanlock: allow more than 8 resources per client
Expand the client tokens array as needed.
SANLK_MAX_RESOURCES remains the limit of resources
explicitly named in a single call.
Signed-off-by: David Teigland <teigland(a)redhat.com>
diff --git a/src/cmd.c b/src/cmd.c
index 809441d..13ad405 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -61,7 +61,7 @@ static void release_cl_tokens(struct task *task, struct client *cl)
struct token *token;
int j;
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
token = cl->tokens[j];
if (!token)
continue;
@@ -92,13 +92,15 @@ static int check_new_tokens_space(struct client *cl,
struct token *token;
int i, rv, empty_slots = 0;
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
if (!cl->tokens[i])
empty_slots++;
}
if (empty_slots < new_tokens_count) {
/* shouldn't ever happen */
+ log_error("check_new_tokens_space slots %d empty %d new_tokens %d",
+ cl->tokens_slots, empty_slots, new_tokens_count);
return -ENOENT;
}
@@ -155,6 +157,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
struct client *cl;
struct token *token = NULL;
struct token *new_tokens[SANLK_MAX_RESOURCES];
+ struct token **grow_tokens;
struct sanlk_resource res;
struct sanlk_options opt;
struct space_info spi;
@@ -168,6 +171,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
int new_tokens_count;
int recv_done = 0;
int result = 0;
+ int grow_size;
int cl_ci = ca->ci_target;
int cl_fd = ca->cl_fd;
int cl_pid = ca->cl_pid;
@@ -195,11 +199,33 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
}
empty_slots = 0;
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
if (!cl->tokens[i])
empty_slots++;
}
+ if (empty_slots < new_tokens_count) {
+ log_debug("cmd_acquire grow tokens slots %d empty %d new %d",
+ cl->tokens_slots, empty_slots, new_tokens_count);
+
+ grow_size = (cl->tokens_slots + (SANLK_MAX_RESOURCES * 2)) * sizeof(struct token *);
+ grow_tokens = malloc(grow_size);
+ if (!grow_tokens) {
+ log_error("cmd_acquire ENOMEM grow tokens slots %d empty %d new %d grow_size %d",
+ cl->tokens_slots, empty_slots, new_tokens_count, grow_size);
+ result = -ENOMEM;
+ pthread_mutex_unlock(&cl->mutex);
+ goto done;
+ } else {
+ memset(grow_tokens, 0, grow_size);
+ memcpy(grow_tokens, cl->tokens, cl->tokens_slots * sizeof(struct token *));
+ free(cl->tokens);
+ cl->tokens = grow_tokens;
+ cl->tokens_slots += (SANLK_MAX_RESOURCES * 2);
+ empty_slots += (SANLK_MAX_RESOURCES * 2);
+ }
+ }
+
memcpy(killpath, cl->killpath, SANLK_HELPER_PATH_LEN);
memcpy(killargs, cl->killargs, SANLK_HELPER_ARGS_LEN);
pthread_mutex_unlock(&cl->mutex);
@@ -452,7 +478,7 @@ static void cmd_acquire(struct task *task, struct cmd_args *ca)
if (!result && !pid_dead) {
for (i = 0; i < new_tokens_count; i++) {
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
if (!cl->tokens[j]) {
cl->tokens[j] = new_tokens[i];
break;
@@ -531,7 +557,7 @@ static void cmd_release(struct task *task, struct cmd_args *ca)
if (ca->header.cmd_flags & SANLK_REL_ALL) {
pthread_mutex_lock(&cl->mutex);
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
token = cl->tokens[j];
if (!token)
continue;
@@ -563,7 +589,7 @@ static void cmd_release(struct task *task, struct cmd_args *ca)
found = 0;
pthread_mutex_lock(&cl->mutex);
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
token = cl->tokens[j];
if (!token)
continue;
@@ -604,7 +630,7 @@ static void cmd_release(struct task *task, struct cmd_args *ca)
found = 0;
pthread_mutex_lock(&cl->mutex);
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
token = cl->tokens[j];
if (!token)
continue;
@@ -658,7 +684,7 @@ static void cmd_release(struct task *task, struct cmd_args *ca)
found = 0;
- for (j = 0; j < SANLK_MAX_RESOURCES; j++) {
+ for (j = 0; j < cl->tokens_slots; j++) {
if (!cl->tokens[j])
continue;
found = 1;
@@ -713,7 +739,7 @@ static void cmd_inquire(struct task *task, struct cmd_args *ca)
goto done;
}
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
if (cl->tokens[i])
res_count++;
}
@@ -734,7 +760,7 @@ static void cmd_inquire(struct task *task, struct cmd_args *ca)
/* should match sanlock_args_to_state() */
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
token = cl->tokens[i];
if (!token)
continue;
@@ -860,7 +886,7 @@ static void cmd_convert(struct task *task, struct cmd_args *ca)
}
pthread_mutex_lock(&cl->mutex);
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
token = cl->tokens[i];
if (!token)
continue;
@@ -2363,6 +2389,20 @@ void call_cmd_daemon(int ci, struct sm_header *h_recv, int client_maxi)
snprintf(client[ci].owner_name, SANLK_NAME_LEN, "%d", pid);
client[ci].pid = pid;
client[ci].deadfn = client_pid_dead;
+
+ if (client[ci].tokens) {
+ log_error("cmd_register ci %d fd %d tokens exist slots %d",
+ ci, fd, client[ci].tokens_slots);
+ free(client[ci].tokens);
+ }
+ client[ci].tokens_slots = SANLK_MAX_RESOURCES;
+ client[ci].tokens = malloc(sizeof(struct token *) * SANLK_MAX_RESOURCES);
+ if (!client[ci].tokens) {
+ rv = -ENOMEM;
+ log_error("cmd_register ci %d fd %d ENOMEM", ci, fd);
+ break;
+ }
+ memset(client[ci].tokens, 0, sizeof(struct token *) * SANLK_MAX_RESOURCES);
auto_close = 0;
break;
case SM_CMD_RESTRICT:
diff --git a/src/main.c b/src/main.c
index b083aa5..421ea67 100644
--- a/src/main.c
+++ b/src/main.c
@@ -260,7 +260,11 @@ static void _client_free(int ci)
memset(cl->killargs, 0, SANLK_HELPER_ARGS_LEN);
cl->workfn = NULL;
cl->deadfn = NULL;
- memset(cl->tokens, 0, sizeof(struct token *) * SANLK_MAX_RESOURCES);
+
+ if (cl->tokens)
+ free(cl->tokens);
+ cl->tokens = NULL;
+ cl->tokens_slots = 0;
/* make poll() ignore this connection */
pollfd[ci].fd = -1;
@@ -497,7 +501,7 @@ void client_pid_dead(int ci)
want to block doing disk lease i/o */
pthread_mutex_lock(&cl->mutex);
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
if (cl->tokens[i]) {
release_token_async(cl->tokens[i]);
free(cl->tokens[i]);
@@ -532,7 +536,7 @@ static int client_using_space(struct client *cl, struct space *sp)
struct token *token;
int i, rv = 0;
- for (i = 0; i < SANLK_MAX_RESOURCES; i++) {
+ for (i = 0; i < cl->tokens_slots; i++) {
token = cl->tokens[i];
if (!token)
continue;
diff --git a/src/sanlock.h b/src/sanlock.h
index 136faa1..58779b1 100644
--- a/src/sanlock.h
+++ b/src/sanlock.h
@@ -10,7 +10,8 @@
#ifndef __SANLOCK_H__
#define __SANLOCK_H__
-/* pid can own this many resources at once */
+/* an acquire or release call can specify this many explicit
+ resources in a single call. */
#define SANLK_MAX_RESOURCES 8
diff --git a/src/sanlock_internal.h b/src/sanlock_internal.h
index a7aeb9b..a448a5c 100644
--- a/src/sanlock_internal.h
+++ b/src/sanlock_internal.h
@@ -225,6 +225,7 @@ struct client {
int suspend;
int need_free;
int kill_count;
+ int tokens_slots;
uint32_t flags;
uint32_t restricted;
uint64_t kill_last;
@@ -234,7 +235,7 @@ struct client {
pthread_mutex_t mutex;
void *workfn;
void *deadfn;
- struct token *tokens[SANLK_MAX_RESOURCES];
+ struct token **tokens;
};
/*
10 years, 5 months
- 1
- 0
10 years, 5 months
- 1
- 0
- ← Newer
- 1
- Older →