5 commits - fedora-ize pkg/cloudfs.spec.in pkg/configure.ac pkg/Makefile.am scripts/cloudfs scripts/volfilter.py xlators/cluster xlators/encryption xlators/features

fedora-ize | 2 pkg/Makefile.am | 2 pkg/cloudfs.spec.in | 21 +---- pkg/configure.ac | 2 scripts/cloudfs | 13 ++- scripts/volfilter.py | 9 +- xlators/cluster/cloud/src/cloud.c | 6 + xlators/encryption/crypt/src/crypt.c | 136 ++++++++++++++++++++++++----------- xlators/encryption/crypt/src/crypt.h | 1 xlators/features/oplock/src/oplock.c | 2 10 files changed, 132 insertions(+), 62 deletions(-) New commits: commit a2786b98842257a29b197e5d416713e01b7c4000 Author: Jeff Darcy <jdarcy(a)redhat.com&gt; Date: Thu Feb 3 17:27:06 2011 -0500 Packaging changes (mostly from rpmlint). diff --git a/fedora-ize b/fedora-ize index c5bca93..a5ac4b9 100755 --- a/fedora-ize +++ b/fedora-ize @@ -20,6 +20,8 @@ cp -r xlators/encryption/crypt/src $work/crypt/ cp -r xlators/features/oplock/src $work/oplock/ cp -r xlators/cluster/login/src $work/login/ cp pkg/* $work/ +cp scripts/cloudfs $work/ +cp scripts/volfilter.py $work/ # Configure just enough to get a decent specfile. cd $work diff --git a/pkg/Makefile.am b/pkg/Makefile.am index 0c83e6c..8fe762f 100644 --- a/pkg/Makefile.am +++ b/pkg/Makefile.am @@ -1,3 +1,5 @@ +bin_SCRIPTS = cloudfs +python_PYTHON = volfilter.py EXTRA_DIST = autogen.sh COPYING cloudfs.spec SUBDIRS = cloud crypt oplock login diff --git a/pkg/cloudfs.spec.in b/pkg/cloudfs.spec.in index 3e0a705..30c8a3f 100644 --- a/pkg/cloudfs.spec.in +++ b/pkg/cloudfs.spec.in @@ -9,14 +9,12 @@ Summary: Cloud File System Name: @PACKAGE_NAME@ Version: @PACKAGE_VERSION@ Release: %{release} -License: AGPLv3+ +License: AGPLv3 +Group: Applications/File Group: System Environment/Base Vendor: Red Hat -Packager: @PACKAGE_BUGREPORT@ URL: http://cloudfs.org -Source0: cloudfs-0.5.tgz -Source1: volfilter.py -Source2: cloudfs +Source0: http://cloudfs.org/dist/0.5/cloudfs-0.5.tgz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root Requires: glusterfs >= 3.1.1 @@ -31,10 +29,6 @@ BuildRequires: openssl-devel CloudFS is a cloud-capable filesystem based on GlusterFS (http://gluster.org) with additional authentication/encryption/multi-tenancy features. -Summary: CloudFS -Group: Applications/File -Provides: cloudfs = %{version}-%{release} - %prep %setup -q -n %{name}-%{version} @@ -48,14 +42,11 @@ Provides: cloudfs = %{version}-%{release} %install %{__rm} -rf %{buildroot} %{__make} install DESTDIR=%{buildroot} -%{__install} -D -p -m 0644 %{SOURCE1} \ - %{buildroot}%{python_sitelib}/volfilter.py -%{__install} -D -p -m 0755 %{SOURCE2} \ - %{buildroot}%{_bindir}/cloudfs # Remove unwanted files from all the shared libraries find %{buildroot}%{_libdir} -name '*.a' -delete find %{buildroot}%{_libdir} -name '*.la' -delete +find %{buildroot}%{_libdir} -name '*.so.0.0.0' | xargs strip %clean %{__rm} -rf %{buildroot} @@ -66,10 +57,10 @@ find %{buildroot}%{_libdir} -name '*.la' -delete %{_libdir}/glusterfs/@GLUSTER_VERSION(a)/xlator/cluster/*.so* %{_libdir}/glusterfs/@GLUSTER_VERSION(a)/xlator/encryption/*.so* %{_libdir}/glusterfs/@GLUSTER_VERSION(a)/xlator/features/*.so* -%{python_sitelib}/volfilter.py +%{python_sitelib}/volfilter.py* %{_bindir}/cloudfs %changelog -* Fri Jan 21 2011 Jeff Darcy <jdarcy(a)redhat.com&gt; - 0.5 +* Fri Jan 21 2011 Jeff Darcy <jdarcy(a)redhat.com&gt; - 0.5-1 - Original version based on GlusterFS 3.1.1 diff --git a/pkg/configure.ac b/pkg/configure.ac index d53fafa..bc64915 100644 --- a/pkg/configure.ac +++ b/pkg/configure.ac @@ -20,6 +20,8 @@ AM_INIT_AUTOMAKE AM_CONFIG_HEADER([config.h]) +AM_PATH_PYTHON([]) + AC_CONFIG_FILES([Makefile cloud/Makefile crypt/Makefile commit 84de7f10d03e37f33b15579e6d42d6aedc571a92 Author: Jeff Darcy <jdarcy(a)redhat.com&gt; Date: Thu Jan 27 20:33:49 2011 -0500 Allow statfs even when unbound. Without this, DHT gets errors back as it's trying to get disk-full information so that it can avoid over-filling a subvolume. This seems more important than the minor amount of information leakage from the pre-binding dummy subvolume (which will in almost all cases be the same actul local filesystem as the real per-tenant subvolumes). diff --git a/xlators/cluster/cloud/src/cloud.c b/xlators/cluster/cloud/src/cloud.c index 5cdbd3f..0bddb3a 100644 --- a/xlators/cluster/cloud/src/cloud.c +++ b/xlators/cluster/cloud/src/cloud.c @@ -192,12 +192,14 @@ cloud_stat (call_frame_t *frame, xlator_t *this, loc_t *loc) return 0; } +#if 0 int32_t cloud_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc) { STACK_UNWIND_STRICT (statfs, frame, -1, EPERM, NULL); return 0; } +#endif int cloud_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath, @@ -495,7 +497,11 @@ struct xlator_fops fops = { .rmdir = cloud_rmdir, .setattr = cloud_setattr, .stat = cloud_stat, + /* + * Allow statfs, because it's fairly harmless and blocking it confuses + * DHT's capacity-balancing code. .statfs = cloud_statfs, + */ .symlink = cloud_symlink, .truncate = cloud_truncate, .unlink = cloud_unlink, commit a0f1109e5bc5bf5e077925ef07d8d34c32d0f733 Author: Jeff Darcy <jdarcy(a)redhat.com&gt; Date: Thu Jan 27 20:31:22 2011 -0500 Insert crypt above write-behind to avoid excessive conflict retries. While conflicts are handled now, write-behind can cause an awful lot of "self conflicts" (where one of two simultaneous writes breaks the other's oplock). This brings the conflict/retry count down to a more reasonable level. diff --git a/scripts/cloudfs b/scripts/cloudfs index d8abe72..4b6615c 100755 --- a/scripts/cloudfs +++ b/scripts/cloudfs @@ -92,6 +92,7 @@ def cloudify_server (volfile, users): for user, pw in users: new_stack = copy_stack(last.subvols[0],user) volfilter.push_filter(graph,new_stack,"features/oplock") + new_stack.name = user subvols.append(new_stack) # One cloud to bring them all... @@ -182,9 +183,14 @@ def do_init_crypt (): for td in to_do: volfilter.delete(graph,td) to_do = [xl for xl in graph.itervalues() - if xl.type == "protocol/client"] - for td in to_do: - volfilter.push_filter(graph,td,"encryption/crypt",opts) + if xl.type == "performance/write-behind"] + if to_do: + for td in to_do: + # Nice to push below io-stats etc. if we can + volfilter.push_filter(graph,td,"encryption/crypt",opts) + else: + # Might as well push it on top. + volfilter.push_filter(graph,last,"encryption/crypt",opts) volfilter.generate(graph,last,file(vfname,"w")) if len(sys.argv) < 3: commit f9a4fd656c857406ae0db0ba541c20454f393211 Author: Jeff Darcy <jdarcy(a)redhat.com&gt; Date: Thu Jan 27 17:33:26 2011 -0500 Implement retries when conflicts are detected. diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c index 6110816..d2d3be1 100644 --- a/xlators/encryption/crypt/src/crypt.c +++ b/xlators/encryption/crypt/src/crypt.c @@ -31,6 +31,13 @@ #include "crypt.h" +/* Forward decls so crypt_launch can retry. */ +int32_t +crypt_rmw_done (call_frame_t *frame, xlator_t *this); +int32_t +crypt_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno); + /* * The "lame" stuff is just for testing (makes it easier to verify correctness * of contents) and must be enabled by hand. @@ -86,7 +93,7 @@ lame_decrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) */ void -good_encrypt (crypt_private_t *priv, char * buf, int len) +good_crypt_buf (crypt_private_t *priv, char * buf, int len, int dir) { DES_cblock ivec; int num; @@ -94,8 +101,9 @@ good_encrypt (crypt_private_t *priv, char * buf, int len) while (len >= priv->block_size) { memset(&ivec,0,sizeof(ivec)); num = 0; - DES_cfb64_encrypt((const unsigned char *)buf,(unsigned char *)buf, - priv->block_size,&priv->sched,&ivec,&num,1); + DES_cfb64_encrypt((const unsigned char *)buf, + (unsigned char *)buf, priv->block_size, + &priv->sched,&ivec,&num,dir); buf += priv->block_size; len -= priv->block_size; } @@ -103,18 +111,25 @@ good_encrypt (crypt_private_t *priv, char * buf, int len) if (len > 0) { memset(&ivec,0,sizeof(ivec)); num = 0; - DES_cfb64_encrypt((const unsigned char *)buf,(unsigned char *)buf, - len,&priv->sched, &ivec,&num,1); + DES_cfb64_encrypt((const unsigned char *)buf, + (unsigned char *)buf,len,&priv->sched,&ivec,&num,dir); } } -/* - * We don't need good_decrypt because good_decrypt_iovec just does the - * looping internally. - */ +void +good_encrypt (crypt_private_t *priv, char * buf, int len) +{ + good_crypt_buf(priv,buf,len,1); +} void -good_decrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) +good_decrypt (crypt_private_t *priv, char * buf, int len) +{ + good_crypt_buf(priv,buf,len,0); +} + +void +good_crypt_iov (crypt_private_t *priv, struct iovec *vector, int count, int dir) { DES_cblock ivec; int num; @@ -137,7 +152,7 @@ good_decrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) } DES_cfb64_encrypt((const unsigned char *)buf, (unsigned char *)buf, - bytes,&priv->sched,&ivec,&num,0); + bytes,&priv->sched,&ivec,&num,dir); buf += bytes; b_resid = (b_resid + bytes) % priv->block_size; if (!b_resid) { @@ -147,6 +162,18 @@ good_decrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) } } +void +good_encrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) +{ + good_crypt_iov(priv,vector,count,1); +} + +void +good_decrypt_iovec (crypt_private_t *priv, struct iovec *vector, int count) +{ + good_crypt_iov(priv,vector,count,0); +} + int32_t crypt_readv_cbk (call_frame_t *frame, void *cookie, @@ -344,6 +371,48 @@ err: } int32_t +crypt_launch (call_frame_t *frame, xlator_t *this) +{ + crypt_wlocal_t *local = frame->local; + crypt_private_t *priv = this->private; + int32_t op_errno = ENOMEM; + + local->call_count = (local->head_resid != 0) + (local->tail_resid != 0); + + /* Check for the only case which doesn't require a lock. */ + if (!local->call_count) { + /* Head and tail were encrypted on the previous pass. */ + good_decrypt(priv,local->head_data,priv->block_size); + good_encrypt(priv,local->head_data,priv->block_size); + return crypt_rmw_done(frame,this); + } + + local->xattr = get_new_dict(); + if (!local->xattr) { + op_errno = ENOMEM; + goto err; + } + + if (dict_set_str(local->xattr,"trusted.glusterfs.lock","fubar") != 0) { + op_errno = EIO; + dict_unref(local->xattr); + goto err; + } + + local->op_ret = 0; + + STACK_WIND (frame, crypt_lock_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->fsetxattr, + local->fd, local->xattr, 0); + + return 0; + +err: + STACK_UNWIND_STRICT(writev,frame,-1,op_errno,NULL,NULL); + return 0; +} + +int32_t crypt_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this, @@ -357,9 +426,15 @@ crypt_writev_cbk (call_frame_t *frame, /* * This is where we might get an error indicating that somebody else * wrote to the file between our read and write. In that case, we - * would simply re-start at the lock call. + * simply re-start at the lock call. */ + if ((op_ret < 0) && (op_errno == EBUSY)) { + gf_log(this->name,GF_LOG_WARNING,"retrying conflicted write"); + local->is_retry = _gf_true; + return crypt_launch(frame,this); + } + if (op_ret > local->orig_size) { op_ret = local->orig_size; } @@ -425,10 +500,12 @@ crypt_rmw_done (call_frame_t *frame, xlator_t *this) b_offset = 0; to_go = local->vector[v_index].iov_len - v_offset; while (to_go >= priv->block_size) { - good_encrypt(priv, - (char *)(local->vector[v_index].iov_base) - + v_offset + b_offset, - priv->block_size); + if (!local->is_retry) { + good_encrypt(priv, + (char *)(local->vector[v_index].iov_base) + + v_offset + b_offset, + priv->block_size); + } b_offset += priv->block_size; to_go -= priv->block_size; } @@ -480,6 +557,7 @@ crypt_rmw_done (call_frame_t *frame, xlator_t *this) goto err; } } + /* * We always start our writes at block boundaries, so we subtract the * "residue" before passing the offset to the next translator. Since @@ -671,31 +749,9 @@ crypt_writev (call_frame_t *frame, } local->head_resid = head_resid; local->tail_resid = tail_resid; - local->call_count = (head_resid != 0) + (tail_resid != 0); - /* Check for the only case which doesn't require a lock. */ - if (!local->call_count) { - return crypt_rmw_done(frame,this); - } - - local->xattr = get_new_dict(); - if (!local->xattr) { - op_errno = ENOMEM; - goto err; - } - - if (dict_set_str(local->xattr,"trusted.glusterfs.lock","fubar") != 0) { - op_errno = EIO; - dict_unref(local->xattr); - goto err; - } - - local->op_ret = 0; - - STACK_WIND (frame, crypt_lock_cbk, FIRST_CHILD(this), - FIRST_CHILD(this)->fops->fsetxattr, fd, local->xattr, 0); - - return 0; + local->is_retry = _gf_false; + return crypt_launch(frame,this); err: STACK_UNWIND_STRICT(writev,frame,-1,op_errno,NULL,NULL); diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h index 0c6548c..fde242b 100644 --- a/xlators/encryption/crypt/src/crypt.h +++ b/xlators/encryption/crypt/src/crypt.h @@ -61,6 +61,7 @@ typedef struct { size_t head_resid; size_t tail_resid; dict_t *xattr; + gf_boolean_t is_retry; } crypt_wlocal_t; #endif /* __CRYPT_H__ */ diff --git a/xlators/features/oplock/src/oplock.c b/xlators/features/oplock/src/oplock.c index 940398e..402e6ea 100644 --- a/xlators/features/oplock/src/oplock.c +++ b/xlators/features/oplock/src/oplock.c @@ -148,6 +148,7 @@ oplock_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vect entry = fetch_op_lock(&priv->locks,inode,state->conn); if (entry) { + LIST_REMOVE(entry,links); if (entry->value != inode->gen) { gf_log(this->name,GF_LOG_DEBUG, "would reject write for %d from %p", @@ -155,7 +156,6 @@ oplock_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vect op_errno = EBUSY; goto err; } - LIST_REMOVE(entry,links); FREE(entry); } ++(inode->gen); commit 5beada814cdbb2bc104669892a322890838fff56 Author: Jeff Darcy <jdarcy(a)redhat.com&gt; Date: Thu Jan 27 17:33:11 2011 -0500 Make slightly less ugly volume names. diff --git a/scripts/cloudfs b/scripts/cloudfs index 30d73bd..d8abe72 100755 --- a/scripts/cloudfs +++ b/scripts/cloudfs @@ -127,7 +127,6 @@ def do_init (): voldir = "" for gdir in glusterd_dirs: probe = "%s/vols/%s" % (gdir, volname) - print "trying %s" % probe if os.access(probe,os.X_OK): voldir = probe break diff --git a/scripts/volfilter.py b/scripts/volfilter.py index 7453c32..98ab20e 100755 --- a/scripts/volfilter.py +++ b/scripts/volfilter.py @@ -85,15 +85,20 @@ def generate (graph, last, stream=sys.stdout): print >> stream, "end-volume" def push_filter (graph, old_xl, filt_type, opts={}): - suffix = string.split(old_xl.type,"/")[1] - new_xl = Translator(old_xl.name+"-"+suffix) + suffix = "-" + old_xl.type.split("/")[1] + if len(old_xl.name) > len(suffix): + if old_xl.name[-len(suffix):] == suffix: + old_xl.name = old_xl.name[:-len(suffix)] + new_xl = Translator(old_xl.name+suffix) new_xl.type = old_xl.type new_xl.opts = old_xl.opts new_xl.subvols = old_xl.subvols graph[new_xl.name] = new_xl + old_xl.name += ("-" + filt_type.split("/")[1]) old_xl.type = filt_type old_xl.opts = opts old_xl.subvols = [new_xl] + graph[old_xl.name] = old_xl def delete (graph, victim): if len(victim.subvols) != 1: