New subject: Change in vdsm[master]: storage.misc: subclass DeferableContext for easy undoing

19 Oct 2012

Zhou Zheng Sheng has uploaded a new change for review.
Change subject: tests: add a rollback manager for easy undoing
......................................................................
tests: add a rollback manager for easy undoing
Sometimes we need to perform a series of operations:
    op[0], op[1], ... op[N]
These operations may allocate files, locks, connections, and op[K] may
depend on op[K-1] 's result
Consider these are contexts, after constructing contexts, we want to
perform some computing using these contexts. Exception may be raised in
the complicated construction stage of the contexts or when we're using
the contexes.
So if op[K] fails, we need to:
    undo op[K-1], undo op[K-2], ... undo op[0]
These undo operations release the resources,
or if all the operations succeed, at last we need to:
    undo op[N], undo op[N-1], ... undo op[0]
Furthermore, We want to suppress the exceptions occured in "undo op[X]",
and continue the rollback, so that all the resources can be freed.
At last, we want to see the first exception raised so that we can fix
the root cause, so we want to have the oldest exception occured in this
batch of operation reraised.
This patch proposes a concise framework to do this kind of
rollback. It's an upgrade version of contextlib.contextmanager .
Change-Id: Ibc932637dd81c3becf92de34ea647c1cea136111
Signed-off-by: Zhou Zheng Sheng zhshzhou@linux.vnet.ibm.com
---
M tests/Makefile.am
A tests/rollbackManagerTests.py
M tests/testrunner.py
3 files changed, 161 insertions(+), 0 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/71/8671/1

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2b61dde..02f48ab 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -45,6 +45,7 @@
    persistentDictTests.py \
    restTests.py \
    restData.py \
+	rollbackManagerTests.py \
    tcTests.py \
    vdsClientTests.py \
    remoteFileHandlerTests.py \
diff --git a/tests/rollbackManagerTests.py b/tests/rollbackManagerTests.py
new file mode 100644
index 0000000..db3a5a0
--- /dev/null
+++ b/tests/rollbackManagerTests.py
@@ -0,0 +1,85 @@
+#
+# Copyright IBM Corp. 2012
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+#
+# Refer to the README and COPYING files for full details of the license
+#
+import glob
+import os
+import tempfile
+import uuid
+
+from testrunner import VdsmTestCase as TestCaseBase
+from testrunner import rollbackManager
+
+
+class ContextError(Exception):
+    pass
+
+
+class ConsumerError(Exception):
+    pass
+
+
+class TestRollbackManager(TestCaseBase):
+    def setUp(self):
+        self.tmpdirPrefix = 'testrollback' + str(uuid.uuid4())
+
+    @rollbackManager
+    def tempfiles(self, fileCount, excClass, rollback):
+        dirPath = tempfile.mkdtemp(prefix=self.tmpdirPrefix)
+        undo = lambda: os.rmdir(dirPath)
+        rollback.append(undo)
+
+        for i in range(0, fileCount):
+            path = os.path.join(dirPath, str(i))
+            with open(path, "wb") as f:
+                undo = \
+                    lambda path=path: os.remove(path)
+                rollback.append(undo)
+                f.write(str(i))
+
+            if excClass is not None:
+                raise excClass("context error")
+
+        return dirPath
+
+    def testExceptionInContext(self):
+        def exceptionInContext():
+            with self.tempfiles(10, ContextError):
+                pass
+
+        self.assertRaises(ContextError, exceptionInContext)
+        # Directory and files should be removed
+        self.assertEquals(glob.glob(self.tmpdirPrefix + "*"), [])
+
+    def testExceptionInConsumer(self):
+        def exceptionInConsumer():
+            with self.tempfiles(10, None):
+                raise ConsumerError("consumer error")
+
+        self.assertRaises(ConsumerError, exceptionInConsumer)
+        # Directory and files should be removed
+        self.assertEquals(glob.glob(self.tmpdirPrefix + "*"), [])
+
+    def testNormalConsumer(self):
+        fileCount = 10
+        with self.tempfiles(fileCount, None) as dirPath:
+            for i in range(0, fileCount):
+                with open(os.path.join(dirPath, str(i)), "rb") as f:
+                    self.assertEquals(int(f.read()), i)
+        # Directory and files should be removed
+        self.assertEquals(glob.glob(self.tmpdirPrefix + "*"), [])
diff --git a/tests/testrunner.py b/tests/testrunner.py
index cdbc9d3..f9cc323 100644
--- a/tests/testrunner.py
+++ b/tests/testrunner.py
@@ -22,6 +22,7 @@
 import os
 import unittest
 from functools import wraps
+from contextlib import contextmanager
from nose import config
 from nose import core
@@ -239,6 +240,80 @@
         return False
+def rollbackManager(transaction):
+    '''
+    A contextmanager-like manager for easy undoing. It's an upgraded
+    contextlib.contextmanager and can manage a variable number of contextes.
+
+    It is used as a decorator to a function. There must exist a parameter
+    named "rollback" of the decorated function. Then the function can treat
+    the "rollback" as a list and append undo operations(lambdas, closures, ...)
+    to the list. The function will be put in the "with" statement, the return
+    value of the function will be assigned to the "as" variable. If an
+    exception is raised, whether it's raised in the function or in the block
+    under the "with" statement, the registered undo operations will be played
+    in reverse order. When peforming rollback, exceptions will be swalloweded
+    to let rollback continue, at last, the earliest exception with original
+    line number and stack trace infomation will be raised.
+
+    Simple example:
+    _________________________________________
+    @rollbackManager
+    def foo(a, b, rollback):
+        x = allocate_X(a)
+        rollback.append(lambda: release_X(x))
+
+        # Need not to catch the exception when allocating Y
+        # and release x in a "try..final" block.
+        # The rollback Manager will do that.
+        y = allocate_Y_using(x)
+        rollback.append(lambda: release_Y(y))
+
+        # Need not to catch the exception when computing z
+        # and release x, y. Let rollback Manager do it for you.
+        z = do_something_with(x, y)
+        return z
+
+    with foo(blah1, blah2) as z:
+        visit(z)
+    _________________________________________
+    When the "with" block is exited, resource y will be released first, then
+    resource x. If exception is raised when constructing y, then only the
+    allocated x will be released, and the original exception will be re-raised.
+    '''
+
+    @contextmanager
+    def wrapper(*args, **kwargs):
+        rollback = []
+        exception = None
+        traceback = None
+        try:
+            yield transaction(rollback=rollback, *args, **kwargs)
+        except Exception as e:
+            # keep the original exception and traceback info
+            exception = e
+            traceback = sys.exc_info()[2]
+        finally:
+            rollback.reverse()
+            _playRollback(rollback, exception, traceback)
+    return wrapper
+
+
+def _playRollback(rollback, exception=None, traceback=None):
+    for undo in rollback:
+        try:
+            undo()
+        except Exception as e:
+            # keep the earliest exception info
+            if not exception:
+                exception = e
+                # keep the original traceback info
+                traceback = sys.exc_info()[2]
+    # re-raise the earliest exception
+    if exception:
+        raise exception, None, traceback
+
+
 if __name__ == '__main__':
     if "--help" in sys.argv:
         print("testrunner options:\n"
--
To view, visit http://gerrit.ovirt.org/8671
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibc932637dd81c3becf92de34ea647c1cea136111
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Zhou Zheng Sheng zhshzhou@linux.vnet.ibm.com

    

Change in vdsm[master]: tests: add a rollback manager for easy undoing