[copr] master: [backend] safer copr_prune_results script, unittests (e539a10)

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master commit e539a10a7a34d3f4ff983704470d30306a132d07 Author: Valentin Gologuzov <vgologuz(a)redhat.com&gt; Date: Tue Mar 24 13:42:09 2015 +0100 [backend] safer copr_prune_results script, unittests backend/backend/createrepo.py | 2 + backend/run/copr_prune_results.py | 28 ++-- backend/tests/_resources/foox.tar.gz | Bin 17135362 -> 6549985 bytes backend/tests/run/test_copr_prune_results.py | 168 ++++++++++++++++++++++++-- 4 files changed, 171 insertions(+), 27 deletions(-) diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 092fa6c..c20656f 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -47,6 +47,8 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None): cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() + # TODO: rewrite with raise CreateRepoException on returncode != 0 + # and return only stdout on success return cmd.returncode, out, err diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index 9f77f81..d9445fd 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -58,20 +58,19 @@ class Pruner(object): log.info("Removing failed build: {}".format(build_path)) shutil.rmtree(build_path) - def prune_obsolete_success_builds(self, chroot_path, username, projectname): + def prune_obsolete_success_builds(self, chroot_path): """ Uses bash script which invokes repoquery to find obsolete build_dirs :param chroot_path: path to the chroot directory - :return: """ # import ipdb; ipdb.set_trace() cmd = map(str, [self.prune_sh_script, chroot_path, self.days]) handle = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = handle.communicate() if handle.returncode != 0: - log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" - .format(username, projectname, stdout.decode(), stderr.decode())) + log.error("Failed to prune old builds at: {} \n STDOUT: \n{}\n STDERR: \n{}\n" + .format(chroot_path, stdout.decode(), stderr.decode())) return # log.debug("find obsolete returned:\n {}\n stderr: \n {}".format(stdout, stderr)) for line in stdout.split("\n"): @@ -121,31 +120,32 @@ class Pruner(object): # run prune project sh for sub_dir_name in os.listdir(project_path): - if sub_dir_name == "repodata": - continue chroot_path = os.path.join(project_path, sub_dir_name) if not os.path.isdir(chroot_path): continue try: self.prune_failed_builds(chroot_path) - self.prune_obsolete_success_builds(chroot_path, username, projectname) + self.prune_obsolete_success_builds(chroot_path) except Exception as err: log.exception(err) - log.error("Error during prune copr {}/{}".format(username, projectname)) + log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name)) - log.debug("Prune done for {}/{}".format(username, projectname)) + log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name)) # run createrepo try: - retcode, stdout, stderr = createrepo_unsafe(project_path) + retcode, stdout, stderr = createrepo_unsafe(chroot_path) if retcode != 0: log.error( - "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" - .format(username, projectname, stdout.decode(), stderr.decode())) + "Failed to createrepo for copr {}/{}:{}\n STDOUT: \n{}\n STDERR: \n{}\n" + .format(username, projectname, sub_dir_name, stdout.decode(), stderr.decode())) + else: + log.info("Createrepo done for copr {}/{}:{}" + .format(username, projectname, sub_dir_name)) except Exception as exception: - log.exception("Createrepo for copr {}/{} failed with error: {}" - .format(username, projectname, exception)) + log.exception("Createrepo for copr {}/{}:{} failed with error: {}" + .format(username, projectname, sub_dir_name, exception)) log.info("Prune finished for copr {}/{}".format(username, projectname)) diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz index aaebe8f..7b3c975 100644 Binary files a/backend/tests/_resources/foox.tar.gz and b/backend/tests/_resources/foox.tar.gz differ diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py index 074e9dc..25c4b80 100644 --- a/backend/tests/run/test_copr_prune_results.py +++ b/backend/tests/run/test_copr_prune_results.py @@ -8,6 +8,7 @@ import tempfile import time from bunch import Bunch from subprocess import Popen, PIPE +from copr.client.exceptions import CoprException import pytest @@ -30,8 +31,28 @@ def mc_popen(): with mock.patch("{}.Popen".format(MODULE_REF)) as handle: yield handle +(a)pytest.yield_fixture +def mc_bcr(): + with mock.patch("{}.BackendConfigReader".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_cru(): + with mock.patch("{}.createrepo_unsafe".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_gacs(): + with mock.patch("{}.get_auto_createrepo_status".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_pruner(): + with mock.patch("{}.Pruner".format(MODULE_REF)) as handle: + yield handle from copr_prune_results import Pruner +from copr_prune_results import main as prune_main def assert_same_dirs(a, b): @@ -42,6 +63,7 @@ def assert_same_dirs(a, b): raise AssertionError("Directories {}, {} aren't equal, diff out: \n {} \n {}" .format(a, b, stdout, stderr)) + class TestPruneResults(object): def setup_method(self, method): @@ -54,22 +76,24 @@ class TestPruneResults(object): self.pkg_2 = "hello-2.8-1.fc20" self.pkg_2_obsolete = "hello-1.0.fc20" self.prj = "foox" - self.chroots = ["fedora-20-i386", "fedora-20-x86_64", - "fedora-21-i386", "fedora-21-x86_64"] + self.chroots = ["fedora-20-i386", "fedora-20-x86_64"] self.opts = Bunch( days=14, find_obsolete_script=os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, - "run", "copr_find_obsolete_builds.sh")) + "run", "copr_find_obsolete_builds.sh")), + + frontend_base_url="http://example.com", + destdir=self.tmp_dir_name ) self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log") logging.basicConfig( - # filename=self.log_file_path, - stream=sys.stdout, + filename=self.log_file_path, + # stream=sys.stdout, format='[%(asctime)s][%(levelname)6s]: %(message)s', level=logging.DEBUG ) @@ -161,10 +185,7 @@ class TestPruneResults(object): os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1_obsolete, "success"), (time.time(), time.time())) - self.pruner.prune_obsolete_success_builds( - os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), - self.username, self.coprname - ) + self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0])) assert_same_dirs( os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), @@ -178,13 +199,134 @@ class TestPruneResults(object): mc_handle.communicate.return_value = ("foo", "bar") # doesn't touch FS if `find_obsolete_build` produce error return code - self.pruner.prune_obsolete_success_builds( - os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), - self.username, self.coprname - ) + self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0])) assert mc_popen.called assert_same_dirs( os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), os.path.join(self.expect_dir_name, self.prj, self.chroots[0]), ) + + def test_prune_project_ok(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + mc_cru.return_value = (0, "", "") + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + expected_path_set = set([ + os.path.join(self.tmp_dir_name, self.prj, chroot) + for chroot in self.chroots + ]) + assert set([ + call[0][0] for call in + self.pruner.prune_failed_builds.call_args_list + ]) == expected_path_set + assert set([ + call[0][0] for call in + self.pruner.prune_obsolete_success_builds.call_args_list + ]) == expected_path_set + assert set([ + call[0][0] for call in + mc_cru.call_args_list + ]) == expected_path_set + + + def test_prune_project_handle_gacs_error(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + + mc_gacs.side_effect = CoprException() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert not self.pruner.prune_failed_builds.called + assert not self.pruner.prune_obsolete_success_builds.called + + def test_prune_project_handle_errors(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + mc_gacs.return_value = True + + + # 0. createrepo_unsafe failure + mc_cru.return_value = (1, "stdout", "stderr") + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert self.pruner.prune_failed_builds.called + assert self.pruner.prune_obsolete_success_builds.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + + mc_cru.return_value = (0, "", "") + # 1. createrepo_unsafe raises error + mc_cru.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert self.pruner.prune_failed_builds.called + assert self.pruner.prune_obsolete_success_builds.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + + mc_cru.side_effect = None + # 2. prune_obsolete_success_builds raises error + + self.pruner.prune_obsolete_success_builds.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert mc_cru.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + mc_cru.reset_mock() + + # 3. prune_failed_builds raises error + + self.pruner.prune_obsolete_success_builds.side_effect = None + self.pruner.prune_failed_builds.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert mc_cru.called + + + def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + + mc_gacs.return_value = False + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert not self.pruner.prune_failed_builds.called + assert not self.pruner.prune_obsolete_success_builds.called + + def test_run(self, test_pruner): + self.pruner.prune_project = MagicMock() + + self.pruner.run() + + + def test_main(self, mc_pruner, mc_bcr): + prune_main() + assert mc_pruner.called + assert mc_pruner.return_value.run.called + assert mc_bcr.called + assert mc_bcr.call_args[0][0] == "/etc/copr/copr-be.conf" + + os.environ["BACKEND_CONFIG"] = "foobar" + prune_main() + assert mc_bcr.call_args[0][0] == "foobar" +