[copr] master: [backend] remove debug stuff from .spec (6b3b67a)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6b3b67adf0c0346fb13fff8222bedc0bfc65c9a2
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 30 18:19:52 2015 +0100
[backend] remove debug stuff from .spec
>---------------------------------------------------------------
backend/copr-backend.spec | 4 ----
1 files changed, 0 insertions(+), 4 deletions(-)
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index a557bce..d7e869d 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -103,10 +103,6 @@ pushd documentation
make %{?_smp_mflags} python
popd
-pushd /tmp
-wget http://people.redhat.com/~vgologuz/permanent/hello_beaker_test_2-0.0.1-1....
-popd
-
%install
install -d %{buildroot}%{_sharedstatedir}/copr
9 years, 2 months
[copr] master: [backend] [rhbz:#1171796] copr sometimes doesn't delete build from repository (d20dc04)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit d20dc04e32784d938d471705ab65b71d2127864e
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 30 14:00:07 2015 +0100
[backend] [rhbz:#1171796] copr sometimes doesn't delete build from repository
>---------------------------------------------------------------
backend/backend/actions.py | 61 ++++---
backend/tests/_resources/1171796.tar.gz | Bin 0 -> 587365 bytes
backend/tests/_resources/1171796_doubled.tar.gz | Bin 0 -> 1172212 bytes
backend/tests/test_action.py | 217 ++++++++++++++++++-----
frontend/run_tests.sh | 4 +-
5 files changed, 207 insertions(+), 75 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index 4ae9c7c..f1d4557 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -39,6 +39,9 @@ class Action(object):
self.front_url = front_url
self.results_root_url = results_root_url
+ def __str__(self):
+ return "<Action: {}>".format(self.data)
+
def add_event(self, what):
self.events.put({"when": time.time(), "who": "action", "what": what})
@@ -101,6 +104,7 @@ class Action(object):
ext_data = json.loads(self.data["data"])
username = ext_data["username"]
projectname = ext_data["projectname"]
+ chroots_requested = set(ext_data["chroots"])
packages = [os.path.basename(x).replace(".src.rpm", "")
for x in ext_data["pkgs"].split()]
@@ -111,34 +115,31 @@ class Action(object):
self.add_event("Copr path {0}".format(path))
try:
- chroot_list = os.listdir(path)
+ chroot_list = set(os.listdir(path))
except OSError:
# already deleted
- chroot_list = []
+ chroot_list = set()
+
+ chroots_to_do = chroot_list.intersection(chroots_requested)
+ if not chroots_to_do:
+ self.add_event("Nothing to delete for delete action: packages {}, {}"
+ .format(packages, ext_data))
+ return
- for chroot in chroot_list:
+ for chroot in chroots_to_do:
self.add_event("In chroot {0}".format(chroot))
altered = False
- # We need to delete the files only if they belong
- # to the build. For example if my build fails and I send
- # fixed pkg with the same version again, it succeeds and
- # than I delete the failed, it would delete the succeeded
- # files as well - that would be wrong.
for pkg in packages:
- if self.data["object_type"] == "build-succeeded" or (
- self.data["object_type"] == "build-failed" and
- os.path.exists(os.path.join(path, chroot, pkg, "fail"))):
-
- pkg_path = os.path.join(path, chroot, pkg)
- if os.path.isdir(pkg_path):
- self.add_event("Removing build {0}".format(pkg_path))
- shutil.rmtree(pkg_path)
- altered = True
- else:
- self.add_event(
- "Package {0} dir not found in chroot {1}"
- .format(pkg, chroot))
+ pkg_path = os.path.join(path, chroot, pkg)
+ if os.path.isdir(pkg_path):
+ self.add_event("Removing build {0}".format(pkg_path))
+ shutil.rmtree(pkg_path)
+ altered = True
+ else:
+ self.add_event(
+ "Package {0} dir not found in chroot {1}"
+ .format(pkg, chroot))
if altered:
self.add_event("Running createrepo")
@@ -154,13 +155,14 @@ class Action(object):
self.add_event(
"Error making local repo: {0}".format(err))
- log_path = os.path.join(
- path, chroot,
- 'build-{0}.log'.format(self.data['object_id']))
-
- if os.path.isfile(log_path):
- self.add_event("Removing log {0}".format(log_path))
- os.unlink(log_path)
+ logs_to_remove = [
+ os.path.join(path, chroot, template.format(self.data['object_id']))
+ for template in ['build-{}.log', 'build-{}.rsync.log']
+ ]
+ for log_path in logs_to_remove:
+ if os.path.isfile(log_path):
+ self.add_event("Removing log {0}".format(log_path))
+ os.remove(log_path)
def run(self):
""" Handle action (other then builds) - like rename or delete of project """
@@ -172,8 +174,7 @@ class Action(object):
if action_type == ActionType.DELETE:
if self.data["object_type"] == "copr":
self.handle_delete_copr_project()
- elif self.data["object_type"] in \
- ["build-succeeded", "build-skipped", "build-failed"]:
+ elif self.data["object_type"] == "build":
self.handle_delete_build()
result.result = ActionResult.SUCCESS
diff --git a/backend/tests/_resources/1171796.tar.gz b/backend/tests/_resources/1171796.tar.gz
new file mode 100644
index 0000000..6d3b522
Binary files /dev/null and b/backend/tests/_resources/1171796.tar.gz differ
diff --git a/backend/tests/_resources/1171796_doubled.tar.gz b/backend/tests/_resources/1171796_doubled.tar.gz
new file mode 100644
index 0000000..6cd04f3
Binary files /dev/null and b/backend/tests/_resources/1171796_doubled.tar.gz differ
diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py
index 24e1367..3e7ce91 100644
--- a/backend/tests/test_action.py
+++ b/backend/tests/test_action.py
@@ -3,6 +3,7 @@ import json
import tempfile
import shutil
import time
+import tarfile
import pytest
import six
@@ -60,7 +61,8 @@ class TestAction(object):
self.ext_data_for_delete_build = json.dumps({
"pkgs": " ".join(self.pkgs),
"username": "foo",
- "projectname": "bar"
+ "projectname": "bar",
+ "chroots": ["fedora20", "epel7"]
})
def teardown_method(self, method):
@@ -83,6 +85,16 @@ class TestAction(object):
return self.tmp_dir_name
+ def unpack_resource(self, resource_name):
+ if self.tmp_dir_name is None:
+ self.make_temp_dir()
+
+ src_path = os.path.join(os.path.dirname(__file__),
+ "_resources", resource_name)
+
+ with tarfile.open(src_path, "r:gz") as tfile:
+ tfile.extractall(os.path.join(self.tmp_dir_name, "old_dir"))
+
def test_action_event(self, mc_time):
test_action = Action(
events=self.test_q,
@@ -308,40 +320,23 @@ class TestAction(object):
tmp_dir = self.make_temp_dir()
- for obj_type in ["build-succeeded", "build-skipped", "build-failed"]:
- test_action = Action(
- action={
- "action_type": ActionType.DELETE,
- "object_type": obj_type,
- "id": 7,
- "old_value": "not-existing-project",
- "data": self.ext_data_for_delete_build,
- },
- events=self.test_q, lock=None,
- frontend_callback=mc_front_cb,
- destdir=tmp_dir,
- front_url=None,
- results_root_url=RESULTS_ROOT_URL
- )
- with mock.patch("backend.actions.shutil") as mc_shutil:
- test_action.run()
- assert not mc_shutil.rmtree.called
-
- ev_1 = self.test_q.get_nowait()
- assert "Action delete build" == ev_1["what"]
- assert ev_1["who"] == "action"
-
- ev_2 = self.test_q.get_nowait()
- assert "Packages to delete" in ev_2["what"]
- assert " ".join(self.pkgs_stripped) in ev_2["what"]
- assert ev_2["who"] == "action"
-
- ev_3 = self.test_q.get_nowait()
- assert "Copr path" in ev_3["what"]
- assert ev_3["who"] == "action"
-
- with pytest.raises(EmptyQueue):
- self.test_q.get_nowait()
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build",
+ "id": 7,
+ "old_value": "not-existing-project",
+ "data": self.ext_data_for_delete_build,
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=tmp_dir,
+ front_url=None,
+ results_root_url=RESULTS_ROOT_URL
+ )
+ with mock.patch("backend.actions.shutil") as mc_shutil:
+ test_action.run()
+ assert not mc_shutil.rmtree.called
@mock.patch("backend.actions.createrepo")
def test_delete_build_succeeded(self, mc_createrepo, mc_time):
@@ -370,7 +365,7 @@ class TestAction(object):
test_action = Action(
action={
"action_type": ActionType.DELETE,
- "object_type": "build-succeeded",
+ "object_type": "build",
"id": 7,
"old_value": "old_dir",
"data": self.ext_data_for_delete_build,
@@ -458,11 +453,11 @@ class TestAction(object):
test_action = Action(
action={
"action_type": ActionType.DELETE,
- "object_type": "build-succeeded",
+ "object_type": "build",
"id": 7,
"old_value": "old_dir",
"data": self.ext_data_for_delete_build,
- "object_id": 42
+ "object_id": 42,
},
events=self.test_q, lock=None,
frontend_callback=mc_front_cb,
@@ -481,11 +476,147 @@ class TestAction(object):
assert error_event_recorded
+ @mock.patch("backend.actions.createrepo")
+ def test_delete_two_chroots(self, mc_createrepo_unsafe, mc_time):
+ """
+ Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796
+
+ """
+ mc_createrepo_unsafe.return_value = 0, STDOUT, ""
+
+ resource_name = "1171796.tar.gz"
+ self.unpack_resource(resource_name)
+
+ chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64")
+ chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64")
+
+ assert os.path.exists(os.path.join(chroot_20_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_21_path, "build-15.log"))
+
+ assert os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log"))
+
+ assert os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21"))
+
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build",
+ "object_id": 15,
+ "id": 15,
+ "old_value": "old_dir",
+ "data": json.dumps({
+ "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm",
+ "username": "foo",
+ "projectname": "bar",
+ "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
+ }),
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=self.tmp_dir_name,
+ front_url=None,
+ results_root_url=RESULTS_ROOT_URL
+ )
+ test_action.run()
+
+ assert not os.path.exists(os.path.join(chroot_20_path, "build-15.log"))
+ assert not os.path.exists(os.path.join(chroot_21_path, "build-15.log"))
+
+ assert not os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log"))
+ assert not os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log"))
+
+ assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21"))
+
+ @mock.patch("backend.actions.createrepo")
+ def test_delete_two_chroots_two_remains(self, mc_createrepo_unsafe, mc_time):
+ """
+ Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796
+ extended: we also put two more chroots, which should be unaffected
+ """
+ mc_createrepo_unsafe.return_value = 0, STDOUT, ""
+
+ resource_name = "1171796_doubled.tar.gz"
+ self.unpack_resource(resource_name)
+
+ chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64")
+ chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64")
+ chroot_20_i386_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-i386")
+ chroot_21_i386_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-i386")
+
+ assert os.path.exists(os.path.join(chroot_20_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_21_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.log"))
+
+ assert os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.rsync.log"))
+
+ assert os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
+
+ mc_time.time.return_value = self.test_time
+ mc_front_cb = MagicMock()
+
+ test_action = Action(
+ action={
+ "action_type": ActionType.DELETE,
+ "object_type": "build",
+ "object_id": 15,
+ "id": 15,
+ "old_value": "old_dir",
+ "data": json.dumps({
+ "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm",
+ "username": "foo",
+ "projectname": "bar",
+ "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
+ }),
+ },
+ events=self.test_q, lock=None,
+ frontend_callback=mc_front_cb,
+ destdir=self.tmp_dir_name,
+ front_url=None,
+ results_root_url=RESULTS_ROOT_URL
+ )
+ test_action.run()
+
+ assert not os.path.exists(os.path.join(chroot_20_path, "build-15.log"))
+ assert not os.path.exists(os.path.join(chroot_21_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.log"))
+ assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.log"))
+
+ assert not os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log"))
+ assert not os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.rsync.log"))
+ assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.rsync.log"))
+
+ assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
+ assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21"))
+
+ @mock.patch("backend.actions.createrepo_unsafe")
+ def test_delete_two_chroots_two_builds_stay_untouched(self, mc_createrepo_unsafe, mc_time):
+ # TODO: prepare archive
+ """
+ Before: 2 builds of the same package-version, all using different chroot
+ """
+ pass
+
@mock.patch("backend.actions.createrepo_unsafe")
- def test_handle_createrepo_ok(self, mc_createrepo_unsfe, mc_time):
+ def test_handle_createrepo_ok(self, mc_createrepo_unsafe, mc_time):
mc_front_cb = MagicMock()
tmp_dir = self.make_temp_dir()
- mc_createrepo_unsfe.return_value = 0, STDOUT, ""
+ mc_createrepo_unsafe.return_value = 0, STDOUT, ""
action_data = json.dumps({
"chroots": ["epel-6-i386", "fedora-20-x86_64"],
@@ -513,9 +644,9 @@ class TestAction(object):
exp_call_1 = mock.call(lock=None, path=tmp_dir + u'/foo/bar/epel-6-i386')
exp_call_2 = mock.call(lock=None, path=tmp_dir + u'/foo/bar/fedora-20-x86_64')
- assert exp_call_1 in mc_createrepo_unsfe.call_args_list
- assert exp_call_2 in mc_createrepo_unsfe.call_args_list
- assert len(mc_createrepo_unsfe.call_args_list) == 2
+ assert exp_call_1 in mc_createrepo_unsafe.call_args_list
+ assert exp_call_2 in mc_createrepo_unsafe.call_args_list
+ assert len(mc_createrepo_unsafe.call_args_list) == 2
@mock.patch("backend.actions.createrepo_unsafe")
diff --git a/frontend/run_tests.sh b/frontend/run_tests.sh
index 41e390e..2ecd9b9 100755
--- a/frontend/run_tests.sh
+++ b/frontend/run_tests.sh
@@ -1,6 +1,6 @@
#!/bin/sh
cd coprs_frontend
-COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests \
- --cov-report term-missing --cov coprs $@
+COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests -s \
+ --cov-report term-missing --cov coprs $@
9 years, 2 months
[copr] master: [frontend] [rhbz:#1171796] copr sometimes doesn't delete build from repository (ea42e4f)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ea42e4f9b871e985db391ee288ac5c8bc98d8fc7
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 30 14:53:07 2015 +0100
[frontend] [rhbz:#1171796] copr sometimes doesn't delete build from repository
>---------------------------------------------------------------
.../coprs_frontend/coprs/logic/builds_logic.py | 41 +++++++-----
.../tests/test_logic/test_builds_logic.py | 73 +++++++++++++++++++-
.../test_views/test_coprs_ns/test_coprs_builds.py | 2 +-
3 files changed, 97 insertions(+), 19 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py
index 5559722..d1cd9b8 100644
--- a/frontend/coprs_frontend/coprs/logic/builds_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py
@@ -212,23 +212,30 @@ class BuildsLogic(object):
"You can not delete build which is not finished.",
"Unfinished build")
- # Only failed (and finished), succeeded, skipped and cancelled get here.
- if build.state != "cancelled": # has nothing in backend to delete
- object_type = "build-{0}".format(build.state)
- data_dict = {"pkgs": build.pkgs,
- "username": build.copr.owner.name,
- "projectname": build.copr.name}
-
- action = models.Action(
- action_type=helpers.ActionTypeEnum("delete"),
- object_type=object_type,
- object_id=build.id,
- old_value="{0}/{1}".format(build.copr.owner.name,
- build.copr.name),
- data=json.dumps(data_dict),
- created_on=int(time.time())
- )
- db.session.add(action)
+ # Only failed, finished, succeeded get here.
+ if build.state not in ["cancelled"]: # has nothing in backend to delete
+ # don't delete skipped chroots
+ chroots_to_delete = [
+ chroot.name for chroot in build.build_chroots
+ if chroot.state not in ["skipped"]
+ ]
+
+ if chroots_to_delete:
+ data_dict = {"pkgs": build.pkgs,
+ "username": build.copr.owner.name,
+ "projectname": build.copr.name,
+ "chroots": chroots_to_delete}
+
+ action = models.Action(
+ action_type=helpers.ActionTypeEnum("delete"),
+ object_type="build",
+ object_id=build.id,
+ old_value="{0}/{1}".format(build.copr.owner.name,
+ build.copr.name),
+ data=json.dumps(data_dict),
+ created_on=int(time.time())
+ )
+ db.session.add(action)
for build_chroot in build.build_chroots:
db.session.delete(build_chroot)
diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
index c7dc07e..ef43cd0 100644
--- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
+++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
@@ -1,10 +1,13 @@
# -*- encoding: utf-8 -*-
+import json
import pytest
import time
+from sqlalchemy.orm.exc import NoResultFound
from coprs import helpers
-from coprs.exceptions import ActionInProgressException
+from coprs.exceptions import ActionInProgressException, InsufficientRightsException
+from coprs.logic.actions_logic import ActionsLogic
from coprs.logic.builds_logic import BuildsLogic
from coprs.logic.builds_logic import BuildsMonitorLogic
@@ -88,3 +91,71 @@ class TestBuildsLogic(CoprsTestCase):
assert len(data) == 1
assert data[0] == self.b1_bc[0]
+
+ def test_delete_build_exceptions(
+ self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+ self.db.session.commit()
+ with pytest.raises(InsufficientRightsException):
+ BuildsLogic.delete_build(self.u1, self.b4)
+
+ self.b1_bc[0].status = "running"
+ self.db.session.add(self.b1_bc[0])
+ self.db.session.commit()
+ with pytest.raises(ActionInProgressException):
+ BuildsLogic.delete_build(self.u1, self.b1)
+
+ def test_delete_build_basic(
+ self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+ self.db.session.commit()
+
+ expected_chroots_to_delete = set()
+ for bchroot in self.b1_bc:
+ expected_chroots_to_delete.add(bchroot.name)
+
+ assert len(ActionsLogic.get_many().all()) == 0
+ BuildsLogic.delete_build(self.u1, self.b1)
+ self.db.session.commit()
+
+ assert len(ActionsLogic.get_many().all()) == 1
+ action = ActionsLogic.get_many().one()
+ delete_data = json.loads(action.data)
+ assert "chroots" in delete_data
+ assert expected_chroots_to_delete == set(delete_data["chroots"])
+
+ with pytest.raises(NoResultFound):
+ BuildsLogic.get(self.b1.id).one()
+
+ def test_delete_build_no_chroots_to_clean(
+ self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+
+ for bchroot in self.b1_bc:
+ bchroot.status = helpers.StatusEnum("skipped")
+
+ self.db.session.commit()
+ assert len(ActionsLogic.get_many().all()) == 0
+ BuildsLogic.delete_build(self.u1, self.b1)
+ self.db.session.commit()
+ assert len(ActionsLogic.get_many().all()) == 0
+
+ def test_delete_build_some_chroots(
+ self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
+
+ expected_chroots_to_delete = set([self.b1_bc[0].name,
+ self.b1_bc[-1].name])
+ for bchroot in self.b1_bc[1:-1]:
+ bchroot.status = helpers.StatusEnum("skipped")
+
+ self.db.session.commit()
+
+ assert len(ActionsLogic.get_many().all()) == 0
+ BuildsLogic.delete_build(self.u1, self.b1)
+ self.db.session.commit()
+
+ assert len(ActionsLogic.get_many().all()) == 1
+ action = ActionsLogic.get_many().one()
+ delete_data = json.loads(action.data)
+ assert "chroots" in delete_data
+ assert expected_chroots_to_delete == set(delete_data["chroots"])
+
+ with pytest.raises(NoResultFound):
+ BuildsLogic.get(self.b1.id).one()
diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
index 889f5e7..6a72158 100644
--- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
+++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py
@@ -118,7 +118,7 @@ class TestCoprDeleteBuild(CoprsTestCase):
.first())
assert b is None
act = self.models.Action.query.first()
- assert act.object_type == "build-succeeded"
+ assert act.object_type == "build"
assert act.old_value == "user1/foocopr"
assert json.loads(act.data)["pkgs"] == pkgs
9 years, 2 months
[copr] master: [backend] [rhbz:#1073333] Record consecutive builds fails to redis. Added script to produce warnings for nagios check from failures recorded to redis. (c66e6e2)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c66e6e28fc58682b24b8a0bbbc6589f2762cac8f
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 30 10:58:05 2015 +0100
[backend] [rhbz:#1073333] Record consecutive builds fails to redis. Added script to produce warnings for nagios check from failures recorded to redis.
>---------------------------------------------------------------
backend/backend/constants.py | 4 ++
backend/backend/daemons/dispatcher.py | 8 ++++
backend/backend/helpers.py | 40 +++++++++++++++++-
backend/run/check_consecutive_build_fails.py | 43 ++++++++++++++++++++
backend/tests/deamons/test_dispatcher.py | 2 +
.../tests/test_logic/test_builds_logic.py | 6 ++-
6 files changed, 99 insertions(+), 4 deletions(-)
diff --git a/backend/backend/constants.py b/backend/backend/constants.py
index e6e01de..717ff27 100644
--- a/backend/backend/constants.py
+++ b/backend/backend/constants.py
@@ -14,6 +14,10 @@ DEF_MACROS = {}
DEF_BUILDROOT_PKGS = ""
+DEF_CONSECUTIVE_FAILURE_THRESHOLD = 10
+CONSECUTIVE_FAILURE_REDIS_KEY = "copr:sys:consecutive_build_fails"
+
+
class BuildStatus(object):
FAILURE = 0
SUCCEEDED = 1
diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py
index 33facc0..2cfb086 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -18,6 +18,8 @@ from setproctitle import setproctitle
from IPy import IP
from retask.queue import Queue
+
+
from ..mockremote.callback import CliLogCallBack
from ..exceptions import MockRemoteError, CoprWorkerError, CoprWorkerSpawnFailError
@@ -26,6 +28,7 @@ from ..job import BuildJob
from ..mockremote import MockRemote
from ..frontend import FrontendClient
from ..constants import BuildStatus
+from ..helpers import register_build_result
ansible_playbook = "ansible-playbook"
@@ -469,6 +472,8 @@ class Worker(multiprocessing.Process):
raise CoprWorkerError(
"No IP found from creating instance")
except AnsibleError as e:
+ register_build_result(self.opts, failed=True)
+
self.callback.log("failure to setup instance: {0}".format(e))
raise
@@ -600,10 +605,13 @@ class Worker(multiprocessing.Process):
if self.opts.do_sign:
mr.add_pubkey()
+ register_build_result(self.opts)
+
except MockRemoteError as e:
# record and break
self.callback.log("{0} - {1}".format(self.vm_ip, e))
status = BuildStatus.FAILURE
+ register_build_result(self.opts, failed=True)
self.callback.log(
"Finished build: id={0} builder={1} timeout={2} destdir={3}"
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 62f7ad9..8554c8b 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -16,12 +16,20 @@ import datetime
from copr.client import CoprClient
-from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT
+from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \
+ CONSECUTIVE_FAILURE_REDIS_KEY
from backend.exceptions import CoprBackendError
+from redis import StrictRedis
+
+try:
+ import fedmsg
+except ImportError:
+ # fedmsg is optional
+ fedmsg = None
-class SortedOptParser(optparse.OptionParser):
+class SortedOptParser(optparse.OptionParser):
"""Optparser which sorts the options by opt before outputting --help"""
def format_help(self, formatter=None):
@@ -146,8 +154,13 @@ class BackendConfigReader(object):
cp, "backend", "sleeptime", 10, mode="int")
opts.timeout = _get_conf(
cp, "builder", "timeout", DEF_BUILD_TIMEOUT, mode="int")
+ opts.consecutive_failure_threshold = _get_conf(
+ cp, "builder", "consecutive_failure_threshold",
+ DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int")
opts.logfile = _get_conf(
cp, "backend", "logfile", "/var/log/copr/backend.log")
+ opts.error_logfile = _get_conf(
+ cp, "backend", "error_logfile", "/var/log/copr/backend_error.log")
opts.verbose = _get_conf(
cp, "backend", "verbose", False, mode="bool")
opts.worker_logdir = _get_conf(
@@ -196,3 +209,26 @@ def log(lf, msg, quiet=None):
"Could not write to logfile {0} - {1}\n".format(lf, str(e)))
if not quiet:
print(msg)
+
+
+def register_build_result(opts=None, failed=False):
+ """
+ Remember fails to redis.
+ Successful build resets counter to zero.
+
+ :param opts: BackendConfig, when opts not provided default config location will be used
+ :param boolean failed: failure flag
+ :param str origin: name of component produced failure, default: `builder`
+ """
+ if opts is None:
+ opts = BackendConfigReader().read()
+
+ # TODO: add config options to specify redis host, port
+ conn = StrictRedis() # connecting to default local redis instance
+
+ key = CONSECUTIVE_FAILURE_REDIS_KEY
+ if not failed:
+ conn.set(key, 0)
+ else:
+ conn.incr(key)
+
diff --git a/backend/run/check_consecutive_build_fails.py b/backend/run/check_consecutive_build_fails.py
new file mode 100755
index 0000000..8e8e26a
--- /dev/null
+++ b/backend/run/check_consecutive_build_fails.py
@@ -0,0 +1,43 @@
+#!/usr/bin/python -tt
+# coding: utf-8
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
+import sys
+
+from redis import StrictRedis
+
+sys.path.append("/usr/share/copr/")
+
+
+from backend.helpers import BackendConfigReader
+from backend.constants import CONSECUTIVE_FAILURE_REDIS_KEY
+
+
+def main():
+ opts = BackendConfigReader().read()
+ conn = StrictRedis() # connecting to default local redis instance
+
+ key = CONSECUTIVE_FAILURE_REDIS_KEY
+
+ value = int(conn.get(key) or 0)
+ if value > opts.consecutive_failure_threshold:
+ print("Critical")
+ sys.exit(2)
+ elif value > int(0.5 * opts.consecutive_failure_threshold):
+ print("Warning")
+ sys.exit(1)
+ else:
+ print("OK")
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except Exception as error:
+ print("UNKNOWN: {}".format(error))
+ sys.exit(3)
diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py
index b50f900..c56bbd4 100644
--- a/backend/tests/deamons/test_dispatcher.py
+++ b/backend/tests/deamons/test_dispatcher.py
@@ -92,6 +92,8 @@ class TestDispatcher(object):
timeout=1800,
destdir=self.tmp_dir_path,
results_baseurl="/tmp",
+
+ consecutive_failure_threshold=10,
)
self.job = BuildJob(self.task, self.opts)
diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
index 1eeb17a..c7dc07e 100644
--- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
+++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py
@@ -74,7 +74,7 @@ class TestBuildsLogic(CoprsTestCase):
def test_build_queue_4(self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
for build_chroots in [self.b1_bc, self.b2_bc]:
for build_chroot in build_chroots:
- build_chroot.status = 3
+ build_chroot.status = 3 # running
for build_chroots in [self.b3_bc, self.b4_bc]:
for build_chroot in build_chroots:
build_chroot.status = 0
@@ -85,4 +85,6 @@ class TestBuildsLogic(CoprsTestCase):
self.db.session.commit()
data = BuildsLogic.get_build_task_queue().all()
- assert len(data) == 1 #
+
+ assert len(data) == 1
+ assert data[0] == self.b1_bc[0]
9 years, 2 months
[copr] master: [frontend] correct url for pubkey in .repo (baf1b06)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit baf1b06286aea5fab7091bc016e87e2f956cb1fa
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Jan 29 21:31:29 2015 +0100
[frontend] correct url for pubkey in .repo
>---------------------------------------------------------------
.../coprs_frontend/coprs/templates/coprs/copr.repo | 2 +-
.../coprs/views/coprs_ns/coprs_general.py | 4 +++-
.../test_views/test_coprs_ns/test_coprs_general.py | 18 +++++++-----------
3 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
index 15e5e7a..d316ae7 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
+++ b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
@@ -3,6 +3,6 @@ name=Copr repo for {{ copr.name }} owned by {{ copr.owner.name }}
baseurl={{ url | fix_url_https_backend }}
skip_if_unavailable=True
gpgcheck=0
-gpgkey={{ url | fix_url_https_backend }}pubkey.gpg
+gpgkey={{ pubkey_url | fix_url_https_backend }}
enabled=1
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 3f8f2c9..be91b69 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -1,5 +1,6 @@
import time
import re
+import urlparse
import flask
from flask import render_template
@@ -545,8 +546,9 @@ def generate_repo_file(username, coprname, chroot, repofile):
.format(username, coprname))
repo_url = generate_repo_url(mock_chroot, url)
+ pubkey_url = urlparse.urljoin(url, "pubkey.gpg")
response = flask.make_response(
- flask.render_template("coprs/copr.repo", copr=copr, url=repo_url))
+ flask.render_template("coprs/copr.repo", copr=copr, url=repo_url, pubkey_url=pubkey_url))
response.mimetype = "text/plain"
response.headers["Content-Disposition"] = \
diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py
index f9d5069..8ecaacc 100644
--- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py
+++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py
@@ -713,17 +713,13 @@ class TestCoprRepoGeneration(CoprsTestCase):
f_custom_builds, f_db):
from coprs import app
orig = app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"]
- try:
- app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = "https"
- r = self.tc.get(
- "/coprs/{0}/{1}/repo/fedora-18-x86_64/"
- .format(self.u1.name, self.c1.name))
-
- assert r.status_code == 200
- assert "baseurl=https://bar.baz" in r.data
- except Exception as e:
- app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig
- raise e
+ app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = "https"
+ r = self.tc.get(
+ "/coprs/{0}/{1}/repo/fedora-18-x86_64/"
+ .format(self.u1.name, self.c1.name))
+
+ assert r.status_code == 200
+ assert "baseurl=https://bar.baz" in r.data
app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig
9 years, 2 months
[copr] master: correctly print job representation (c6e2cb9)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c6e2cb9cf0d65a16a937d301b51f8cb19dae9f74
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Mon Jan 26 13:28:53 2015 +0100
correctly print job representation
addressing:
Process worker-builder:
Traceback (most recent call last):
File "/usr/lib64/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/share/copr/backend/daemons/dispatcher.py", line 662, in run
job = self.obtain_job()
File "/usr/share/copr/backend/daemons/dispatcher.py", line 520, in obtain_job
self.callback.log("obtained task: {}".format(job))
File "/usr/share/copr/backend/job.py", line 111, in __str__
return str(self.__unicode__())
File "/usr/share/copr/backend/job.py", line 115, in __unicode__
u"project: {project_name}, source pkg : {pkg} >".format(self.__dict__)
KeyError: u'build_id'
>---------------------------------------------------------------
backend/backend/job.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/backend/job.py b/backend/backend/job.py
index 6d07b26..232ec09 100644
--- a/backend/backend/job.py
+++ b/backend/backend/job.py
@@ -112,4 +112,4 @@ class BuildJob(object):
def __unicode__(self):
return u"BuildJob<id: {build_id}, owner: {project_owner}, " \
- u"project: {project_name}, source pkg : {pkg} >".format(self.__dict__)
+ u"project: {project_name}, source pkg : {pkg} >".format(**self.__dict__)
9 years, 3 months
[copr] master: Automatic commit of package [copr-backend] release [1.57-1]. (ecdb0b4)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ecdb0b4bbad9310ff6645dc3f563067156bb6bd7
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 23 15:52:40 2015 +0100
Automatic commit of package [copr-backend] release [1.57-1].
>---------------------------------------------------------------
backend/copr-backend.spec | 26 +++++++++++++++++++++++++-
rel-eng/packages/copr-backend | 2 +-
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index ac29eaa..a557bce 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -3,7 +3,7 @@
%endif
Name: copr-backend
-Version: 1.56
+Version: 1.57
Release: 1%{?dist}
Summary: Backend for Copr
@@ -211,6 +211,30 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%exclude %{_pkgdocdir}/playbooks
%changelog
+* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.57-1
+- call correct Worker method on backend termination
+- put gpg pubkey to the project results root dir (one level up from
+ the chroot dir)
+- don't kill Worker from errors during job build
+- [rhbz:#1169782] RFE - Show package "version-release" instead of
+ just "version"
+- [rhbz:#1117446] add a build id tagfile into the package directory
+- Updated unittests to reflect latest changes.
+- builder: use only one log file for rsync per build
+- dispatcher: run terminate_instance safely
+- cleanup example config
+- cleanup mockremote.builder
+- Builder.download don't use Popen+PIPE.communicate with rsync,
+ output redirected to the files.
+- disable networking only when required; python style exception
+ handling in mockremote*; removed run/copr_mockremote
+- test build with disabled networking
+- simplified mockremote.builder.Builder.check_for_ans_error; new
+ method mockremote.builder.Builder.run_ansible_with_check
+- daemons.dispatched.Worker: don't fail when wrong group_id was
+ provided
+- add vm_ip to worker process title (rhbz: 1182637)
+
* Wed Jan 14 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.56-1
- [backend] [.spec] fix %files section
diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend
index ffdd123..34dea39 100644
--- a/rel-eng/packages/copr-backend
+++ b/rel-eng/packages/copr-backend
@@ -1 +1 @@
-1.56-1 backend/
+1.57-1 backend/
9 years, 3 months
[copr] master: Automatic commit of package [copr-frontend] release [1.52-1]. (a137935)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit a1379358cbf41dbb09e27b661d9f2589bd38b005
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Jan 23 15:49:45 2015 +0100
Automatic commit of package [copr-frontend] release [1.52-1].
>---------------------------------------------------------------
frontend/copr-frontend.spec | 15 ++++++++++++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index 420972f..43e88ac 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.51
+Version: 1.52
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -168,6 +168,19 @@ service httpd condrestart
%doc documentation/python-doc
%changelog
+* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.52-1
+- add url to gpg pubkey in .repo files
+- [rhbz:#1183702] Interrupted builds aren't re-added to the
+ builder queue, and stuck forever in RUNNING state.
+- [rhbz:#1133650] RFE: copr frontend on page of build details,
+ results section should show multiple links that link directly for every
+ chroot directory
+- UI to control `enable_net` option, DB schema changes
+- new command AddDebugUser for manage script
+- [RHBZ:#1176364] Wrong value for the build timeout.
+- [RHBZ:#1177179] Display the timezone with a format more similar to
+ ISO 8601
+
* Mon Dec 15 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.51-1
- bugfix: send correct chroots in on_auto_createrepo_change()
- control auto_createrepo property of project through API
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 3a85546..9d79b41 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.51-1 frontend/
+1.52-1 frontend/
9 years, 3 months