[copr] master: [backend] eliminate global multiprocessing.Lock (86a6637)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 86a663772910d7eabbe7a983c9029a195f796ec5
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Sep 11 18:52:39 2015 +0200
[backend] eliminate global multiprocessing.Lock
>---------------------------------------------------------------
backend/backend/actions.py | 14 ++---
backend/backend/createrepo.py | 33 ++++-----
backend/backend/daemons/job_grab.py | 4 +-
backend/backend/mockremote/__init__.py | 1 -
backend/copr-backend.spec | 5 +-
backend/run/systemd/copr-backend.service | 2 +-
backend/tests/deamons/test_backend.py | 22 +------
backend/tests/deamons/test_job_grab.py | 2 +-
backend/tests/mockremote/test_mockremote.py | 1 -
backend/tests/test_action.py | 39 ++---------
backend/tests/test_createrepo.py | 94 +++++++++------------------
11 files changed, 64 insertions(+), 153 deletions(-)
diff --git a/backend/backend/actions.py b/backend/backend/actions.py
index 7548757..dc3c35d 100644
--- a/backend/backend/actions.py
+++ b/backend/backend/actions.py
@@ -6,15 +6,14 @@ from urllib import urlretrieve
from munch import Munch
-from .createrepo import createrepo, createrepo_unsafe
-from exceptions import CreateRepoError
+from .createrepo import createrepo
+from .exceptions import CreateRepoError
from .helpers import get_redis_logger, silent_remove
class Action(object):
""" Object to send data back to fronted
- :param multiprocessing.Lock lock: Global lock for backend
:param backend.callback.FrontendCallback frontent_callback:
object to post data back to frontend
@@ -29,14 +28,12 @@ class Action(object):
"""
# TODO: get more form opts, decrease number of parameters
- def __init__(self, opts, action, lock, frontend_client):
+ def __init__(self, opts, action, frontend_client):
self.opts = opts
self.frontend_client = frontend_client
self.data = action
- self.lock = lock
-
self.destdir = self.opts.destdir
self.front_url = self.opts.frontend_base_url
self.results_root_url = self.opts.results_baseurl
@@ -66,8 +63,7 @@ class Action(object):
try:
createrepo(path=path, front_url=self.front_url,
username=username, projectname=projectname,
- override_acr_flag=True,
- lock=self.lock)
+ override_acr_flag=True)
done_count += 1
except CreateRepoError:
self.log.exception("Error making local repo for: {}/{}/{}"
@@ -194,7 +190,7 @@ class Action(object):
createrepo_target = os.path.join(path, chroot)
try:
createrepo(
- path=createrepo_target, lock=self.lock,
+ path=createrepo_target,
front_url=self.front_url, base_url=result_base_url,
username=username, projectname=projectname
)
diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py
index 8a36e57..e6520e4 100644
--- a/backend/backend/createrepo.py
+++ b/backend/backend/createrepo.py
@@ -2,8 +2,8 @@ import os
import subprocess
from subprocess import Popen, PIPE
-from exceptions import CreateRepoError
from shlex import split
+from lockfile import LockFile
# todo: add logging here
# from backend.helpers import BackendConfigReader, get_redis_logger
@@ -11,20 +11,17 @@ from shlex import split
# log = get_redis_logger(opts, "createrepo", "actions")
from .helpers import get_auto_createrepo_status
+from .exceptions import CreateRepoError
-def run_cmd_unsafe(comm_str, lock=None):
+def run_cmd_unsafe(comm_str, lock_path):
# log.info("Running command: {}".format(comm_str))
comm = split(comm_str)
try:
- # # todo: replace with file lock on target dir
- if lock:
- with lock:
- cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = cmd.communicate()
- else:
+ with LockFile(lock_path):
cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = cmd.communicate()
+
except Exception as err:
raise CreateRepoError(msg="Failed to execute: {}".format(err), cmd=comm_str)
@@ -35,7 +32,7 @@ def run_cmd_unsafe(comm_str, lock=None):
return out
-def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
+def createrepo_unsafe(path, dest_dir=None, base_url=None):
"""
Run createrepo_c on the given path
@@ -73,7 +70,7 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None):
comm.append(path)
- return run_cmd_unsafe(" ".join(map(str, comm)), lock)
+ return run_cmd_unsafe(" ".join(map(str, comm)), os.path.join(path, "createrepo.lock"))
APPDATA_CMD_TEMPLATE = \
@@ -115,20 +112,20 @@ def add_appdata(path, username, projectname, lock=None):
}
try:
out += "\n" + run_cmd_unsafe(
- APPDATA_CMD_TEMPLATE.format(**kwargs), lock)
+ APPDATA_CMD_TEMPLATE.format(**kwargs), os.path.join(path, "createrepo.lock"))
if os.path.exists(os.path.join(path, "appdata", "appstream.xml.gz")):
out += "\n" + run_cmd_unsafe(
- INCLUDE_APPSTREAM.format(**kwargs), lock)
+ INCLUDE_APPSTREAM.format(**kwargs), os.path.join(path, "createrepo.lock"))
if os.path.exists(os.path.join(path, "appdata", "appstream-icons.tar.gz")):
out += "\n" + run_cmd_unsafe(
- INCLUDE_ICONS.format(**kwargs), lock)
+ INCLUDE_ICONS.format(**kwargs), os.path.join(path, "createrepo.lock"))
# appstream builder provide strange access rights to result dir
# fix them, so that lighttpd could serve appdata dir
out += "\n" + run_cmd_unsafe("chmod -R +rX {packages_dir}"
- .format(**kwargs), lock)
+ .format(**kwargs), os.path.join(path, "createrepo.lock"))
except CreateRepoError as err:
err.stdout = out + "\nLast command\n" + err.stdout
raise
@@ -136,7 +133,7 @@ def add_appdata(path, username, projectname, lock=None):
def createrepo(path, front_url, username, projectname,
- override_acr_flag=False, base_url=None, lock=None):
+ override_acr_flag=False, base_url=None):
"""
Creates repo depending on the project setting "auto_createrepo".
When enabled creates `repodata` at the provided path, otherwise
@@ -156,8 +153,8 @@ def createrepo(path, front_url, username, projectname,
acr_flag = get_auto_createrepo_status(front_url, username, projectname)
if override_acr_flag or acr_flag:
- out_cr = createrepo_unsafe(path, lock)
- out_ad = add_appdata(path, username, projectname, lock)
+ out_cr = createrepo_unsafe(path)
+ out_ad = add_appdata(path, username, projectname)
return "\n".join([out_cr, out_ad])
else:
- return createrepo_unsafe(path, lock, base_url=base_url, dest_dir="devel")
+ return createrepo_unsafe(path, base_url=base_url, dest_dir="devel")
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index 92d8504..df6e98e 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -58,7 +58,7 @@ class CoprJobGrab(Process):
self.task_queues_by_group = {}
self.added_jobs_dict = dict() # task_id -> task dict
- self.lock = lock
+ self.lock = lock # used only for createrepo calls
self.frontend_client = frontend_client
@@ -139,7 +139,7 @@ class CoprJobGrab(Process):
:param action: dict-like object with action task
"""
- ao = Action(self.opts, action, self.lock, frontend_client=self.frontend_client)
+ ao = Action(self.opts, action, frontend_client=self.frontend_client)
ao.run()
def load_tasks(self):
diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py
index aec2081..ceffc28 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -243,7 +243,6 @@ class MockRemote(object):
base_url=base_url,
username=self.job.project_owner,
projectname=self.job.project_name,
- lock=self.lock,
)
except CreateRepoError:
self.log.exception("Error making local repo: {}".format(self.chroot_dir))
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 3ddd257..bb8fe0e 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -28,16 +28,14 @@ BuildRequires: python-copr
BuildRequires: systemd
BuildRequires: redis
-
-
BuildRequires: pytest
BuildRequires: python-pytest-cov
BuildRequires: python-mock
BuildRequires: python-six
BuildRequires: python-munch
+BuildRequires: python-lockfile
# missing python3
BuildRequires: python-daemon
-BuildRequires: python-lockfile
BuildRequires: python-requests
BuildRequires: python-setproctitle
# missing python3
@@ -85,6 +83,7 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
+Requires: python-lockfile
# Requires: python-ipdb
Requires: logstash
Requires: libappstream-glib-builder >= 0.4.0
diff --git a/backend/run/systemd/copr-backend.service b/backend/run/systemd/copr-backend.service
index ea43924..ce3a4ef 100644
--- a/backend/run/systemd/copr-backend.service
+++ b/backend/run/systemd/copr-backend.service
@@ -1,5 +1,5 @@
[Unit]
-Description=Copr Backend service, Virtual Machine Management component
+Description=Copr Backend service, Workers controller
After=syslog.target network.target auditd.service
Requires=copr-backend-vmm.service
Wants=copr-backend-log.service
diff --git a/backend/tests/deamons/test_backend.py b/backend/tests/deamons/test_backend.py
index 37fade0..7d047d4 100644
--- a/backend/tests/deamons/test_backend.py
+++ b/backend/tests/deamons/test_backend.py
@@ -124,19 +124,6 @@ class TestBackend(object):
self.be = CoprBackend(self.config_file, self.ext_opts)
self.be.log = MagicMock()
- @pytest.yield_fixture
- def mc_vmm_stuff(self):
- patchers = []
- for klass_name in ["Spawner", "HealthChecker", "Terminator",
- "VmManager", "VmMaster", "EventHandler"]:
- patcher = mock.patch("{}.{}".format(MODULE_REF, klass_name))
- patchers.append(patcher)
- setattr(self, klass_name, patcher.start())
-
- yield None
- for patcher in patchers:
- patcher.stop()
-
def teardown_method(self, method):
# print("\nremove: {}".format(self.tmp_dir_path))
shutil.rmtree(self.tmp_dir_path)
@@ -192,9 +179,8 @@ class TestBackend(object):
with pytest.raises(CoprBackendError):
self.be.init_task_queues()
- @mock.patch("backend.daemons.backend.RedisLogHandler")
@mock.patch("backend.daemons.backend.CoprJobGrab")
- def test_dummy_init_sub_process(self, mc_jobgrab, mc_rlh, init_be, mc_vmm_stuff):
+ def test_dummy_init_sub_process(self, mc_jobgrab, init_be):
self.be.init_sub_process()
assert mc_jobgrab.called
@@ -203,12 +189,6 @@ class TestBackend(object):
lock=self.be.lock)
assert mc_jobgrab.return_value.start.called
- assert mc_rlh.called
- assert mc_rlh.return_value.start.called
-
- assert self.VmMaster.called
- assert self.VmMaster.return_value.start.called
-
def test_update_conf(self, init_be):
test_obj = MagicMock()
self.bc_obj.read.return_value = test_obj
diff --git a/backend/tests/deamons/test_job_grab.py b/backend/tests/deamons/test_job_grab.py
index 0ff8068..b1cf828 100644
--- a/backend/tests/deamons/test_job_grab.py
+++ b/backend/tests/deamons/test_job_grab.py
@@ -220,7 +220,7 @@ class TestJobGrab(object):
self.jg.process_action(test_action)
- expected_call = call(self.opts, test_action, self.lock, frontend_client=self.jg.frontend_client)
+ expected_call = call(self.opts, test_action, frontend_client=self.jg.frontend_client)
assert expected_call == mc_action.call_args
assert mc_action.return_value.run.called
diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py
index fb13c13..ff0aabc 100644
--- a/backend/tests/mockremote/test_mockremote.py
+++ b/backend/tests/mockremote/test_mockremote.py
@@ -146,7 +146,6 @@ class TestMockRemote(object):
base_url=u"/".join([self.BASE_URL, COPR_OWNER, COPR_NAME, self.CHROOT]),
username=COPR_OWNER,
projectname=COPR_NAME,
- lock=None,
)
assert mc_createrepo.call_args == expected_call
diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py
index 252d39d..e029740 100644
--- a/backend/tests/test_action.py
+++ b/backend/tests/test_action.py
@@ -93,7 +93,6 @@ class TestAction(object):
"action_type": ActionType.LEGAL_FLAG,
"id": 1
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -119,7 +118,6 @@ class TestAction(object):
"old_value": "old_dir",
"new_value": "new_dir"
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -150,7 +148,6 @@ class TestAction(object):
"old_value": "old_dir",
"new_value": "new_dir"
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -180,7 +177,6 @@ class TestAction(object):
"old_value": "old_dir",
"new_value": "new_dir"
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -210,7 +206,6 @@ class TestAction(object):
"id": 6,
"old_value": "old_dir",
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -236,7 +231,6 @@ class TestAction(object):
"id": 6,
"old_value": "old_dir",
},
- lock=None,
frontend_client=mc_front_cb
)
test_action.run()
@@ -263,7 +257,6 @@ class TestAction(object):
"old_value": "not-existing-project",
"data": self.ext_data_for_delete_build,
},
- lock=None,
frontend_client=mc_front_cb
)
with mock.patch("backend.actions.shutil") as mc_shutil:
@@ -305,7 +298,6 @@ class TestAction(object):
"data": self.ext_data_for_delete_build,
"object_id": 42
},
- lock=None,
frontend_client=mc_front_cb,
)
@@ -320,7 +312,6 @@ class TestAction(object):
username=u'foo',
projectname=u'bar',
base_url=u'http://example.com/results/foo/bar/fedora20',
- lock=None,
path='{}/old_dir/fedora20'.format(self.tmp_dir_name),
front_url=None
)
@@ -357,19 +348,18 @@ class TestAction(object):
"data": self.ext_data_for_delete_build,
"object_id": 42,
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@mock.patch("backend.actions.createrepo")
- def test_delete_two_chroots(self, mc_createrepo_unsafe, mc_time):
+ def test_delete_two_chroots(self, mc_createrepo, mc_time):
"""
Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796
"""
- mc_createrepo_unsafe.return_value = 0, STDOUT, ""
+ mc_createrepo.return_value = 0, STDOUT, ""
resource_name = "1171796.tar.gz"
self.unpack_resource(resource_name)
@@ -389,7 +379,7 @@ class TestAction(object):
mc_time.time.return_value = self.test_time
mc_front_cb = MagicMock()
- self.opts.destdir=self.tmp_dir_name
+ self.opts.destdir = self.tmp_dir_name
test_action = Action(
opts=self.opts,
action={
@@ -405,7 +395,6 @@ class TestAction(object):
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
}),
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -423,12 +412,12 @@ class TestAction(object):
assert os.path.exists(chroot_21_path)
@mock.patch("backend.actions.createrepo")
- def test_delete_two_chroots_two_remains(self, mc_createrepo_unsafe, mc_time):
+ def test_delete_two_chroots_two_remains(self, mc_createrepo, mc_time):
"""
Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796
extended: we also put two more chroots, which should be unaffected
"""
- mc_createrepo_unsafe.return_value = 0, STDOUT, ""
+ mc_createrepo.return_value = 0, STDOUT, ""
resource_name = "1171796_doubled.tar.gz"
self.unpack_resource(resource_name)
@@ -472,7 +461,6 @@ class TestAction(object):
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
}),
},
- lock=None,
frontend_client=mc_front_cb
)
test_action.run()
@@ -524,7 +512,6 @@ class TestAction(object):
"chroots": ["fedora-20-x86_64", "fedora-21-x86_64"]
}),
},
- lock=None,
frontend_client=mc_front_cb,
)
@@ -535,15 +522,6 @@ class TestAction(object):
assert os.path.exists(chroot_20_path)
assert os.path.exists(chroot_21_path)
-
- @mock.patch("backend.actions.createrepo_unsafe")
- def test_delete_two_chroots_two_builds_stay_untouched(self, mc_createrepo_unsafe, mc_time):
- # TODO: prepare archive
- """
- Before: 2 builds of the same package-version, all using different chroot
- """
- pass
-
@mock.patch("backend.actions.createrepo")
def test_handle_createrepo_ok(self, mc_createrepo, mc_time):
mc_front_cb = MagicMock()
@@ -563,7 +541,6 @@ class TestAction(object):
"data": action_data,
"id": 8
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -575,10 +552,10 @@ class TestAction(object):
exp_call_1 = mock.call(path=tmp_dir + u'/foo/bar/epel-6-i386',
front_url=self.opts.frontend_base_url, override_acr_flag=True,
- username=u"foo", projectname=u"bar", lock=None)
+ username=u"foo", projectname=u"bar")
exp_call_2 = mock.call(path=tmp_dir + u'/foo/bar/fedora-20-x86_64',
front_url=self.opts.frontend_base_url, override_acr_flag=True,
- username=u"foo", projectname=u"bar", lock=None)
+ username=u"foo", projectname=u"bar")
assert exp_call_1 in mc_createrepo.call_args_list
assert exp_call_2 in mc_createrepo.call_args_list
assert len(mc_createrepo.call_args_list) == 2
@@ -603,7 +580,6 @@ class TestAction(object):
"data": action_data,
"id": 9
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
@@ -635,7 +611,6 @@ class TestAction(object):
"data": action_data,
"id": 10
},
- lock=None,
frontend_client=mc_front_cb,
)
test_action.run()
diff --git a/backend/tests/test_createrepo.py b/backend/tests/test_createrepo.py
index d982b8f..ec1bba7 100644
--- a/backend/tests/test_createrepo.py
+++ b/backend/tests/test_createrepo.py
@@ -29,14 +29,14 @@ def test_createrepo_conditional_true(mc_client, mc_add_appdata, mc_create_unsafe
mc_add_appdata.return_value = ""
createrepo(path="/tmp/", front_url="http://example.com/api",
- username="foo", projectname="bar", lock=None)
+ username="foo", projectname="bar")
mc_create_unsafe.reset_mock()
mc_client.return_value.get_project_details.return_value = MagicMock(
data={"detail": {"auto_createrepo": True}})
createrepo(path="/tmp/", front_url="http://example.com/api",
- username="foo", projectname="bar", lock=None)
+ username="foo", projectname="bar")
mc_create_unsafe.reset_mock()
@@ -48,9 +48,9 @@ def test_createrepo_conditional_false(mc_client, mc_create_unsafe):
base_url = "http://example.com/repo/"
createrepo(path="/tmp/", front_url="http://example.com/api",
- username="foo", projectname="bar", base_url=base_url, lock=None)
+ username="foo", projectname="bar", base_url=base_url)
- assert mc_create_unsafe.call_args == mock.call('/tmp/', None, dest_dir='devel', base_url=base_url)
+ assert mc_create_unsafe.call_args == mock.call('/tmp/', dest_dir='devel', base_url=base_url)
@pytest.yield_fixture
@@ -111,38 +111,35 @@ class TestCreaterepo(object):
mc_popen.return_value.communicate.return_value = ("stdout", "stderr")
mc_popen.return_value.returncode = 0
- for lock in [None, MagicMock()]:
- assert run_cmd_unsafe(cmd, lock) == "stdout"
- assert mc_popen.call_args[0][0] == ["foo", "--bar"]
- mc_popen.reset()
+ assert run_cmd_unsafe(cmd, self.tmp_dir_name) == "stdout"
+ mc_popen.reset()
def test_run_cmd_unsafe_err_popen(self, mc_popen):
cmd = "foo --bar"
mc_popen.side_effect = IOError()
- for lock in [None, MagicMock()]:
- with pytest.raises(CreateRepoError) as err:
- run_cmd_unsafe(cmd, lock) == "stdout"
+ with pytest.raises(CreateRepoError) as err:
+ run_cmd_unsafe(cmd, self.tmp_dir_name) == "stdout"
- assert err.value.cmd == cmd
- assert mc_popen.call_args[0][0] == ["foo", "--bar"]
- mc_popen.reset()
+ assert err.value.cmd == cmd
+ assert mc_popen.call_args[0][0] == ["foo", "--bar"]
+ mc_popen.reset()
def test_run_cmd_unsafe_err_return_code(self, mc_popen):
cmd = "foo --bar"
mc_popen.return_value.communicate.return_value = ("stdout", "stderr")
mc_popen.return_value.returncode = 1
- for lock in [None, MagicMock()]:
- with pytest.raises(CreateRepoError) as err:
- run_cmd_unsafe(cmd, lock) == "stdout"
- assert err.value.cmd == cmd
- assert err.value.stdout == "stdout"
- assert err.value.stderr == "stderr"
- assert err.value.exit_code == 1
- assert mc_popen.call_args[0][0] == ["foo", "--bar"]
- mc_popen.reset()
+ with pytest.raises(CreateRepoError) as err:
+ run_cmd_unsafe(cmd, self.tmp_dir_name) == "stdout"
+
+ assert err.value.cmd == cmd
+ assert err.value.stdout == "stdout"
+ assert err.value.stderr == "stderr"
+ assert err.value.exit_code == 1
+ assert mc_popen.call_args[0][0] == ["foo", "--bar"]
+ mc_popen.reset()
def test_run_cmd_unsafe_err_communicate(self, mc_popen):
cmd = "foo --bar"
@@ -151,42 +148,12 @@ class TestCreaterepo(object):
mc_handle.returncode = 0
mc_handle.side_effect = RuntimeError()
- for lock in [None, MagicMock()]:
- with pytest.raises(CreateRepoError) as err:
- run_cmd_unsafe(cmd, lock) == "stdout"
-
- assert err.value.cmd == cmd
- assert mc_popen.call_args[0][0] == ["foo", "--bar"]
- mc_popen.reset()
-
- def test_run_cmd_unsafe_lock_usage(self, mc_popen):
- cmd = "foo --bar"
- mocked_lock = MagicMock()
- self.shared_state = dict(in_lock=False, lock_status=None)
-
- def enter_lock(*args, **kwargs):
- self.shared_state["in_lock"] = True
-
- def exit_lock(*args, **kwargs):
- self.shared_state["in_lock"] = False
+ with pytest.raises(CreateRepoError) as err:
+ run_cmd_unsafe(cmd, self.tmp_dir_name) == "stdout"
- def popen_side_effect(*args, **kwargs):
- self.shared_state["lock_status"] = copy.copy(self.shared_state["in_lock"])
- return mock.DEFAULT
-
- mocked_lock.__enter__.side_effect = enter_lock
- mocked_lock.__exit__.side_effect = exit_lock
-
- mc_popen.side_effect = popen_side_effect
- mc_popen.return_value.communicate.return_value = ("", "")
- mc_popen.return_value.returncode = 0
-
- run_cmd_unsafe(cmd, lock=mocked_lock)
- assert self.shared_state["lock_status"]
-
- self.shared_state["lock_status"] = None
- run_cmd_unsafe(cmd, lock=None)
- assert not self.shared_state["lock_status"]
+ assert err.value.cmd == cmd
+ assert mc_popen.call_args[0][0] == ["foo", "--bar"]
+ mc_popen.reset()
def test_createrepo_generated_commands_existing_repodata(self, mc_run_cmd_unsafe):
path_epel_5 = os.path.join(self.tmp_dir_name, "epel-5")
@@ -203,7 +170,7 @@ class TestCreaterepo(object):
with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
handle.write("1")
- createrepo_unsafe(path, None)
+ createrepo_unsafe(path)
assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_generated_commands_comps_xml(self, mc_run_cmd_unsafe):
@@ -218,13 +185,12 @@ class TestCreaterepo(object):
with open(comps_path, "w") as handle:
handle.write("1")
-
repo_path = os.path.join(path, "repodata")
os.makedirs(repo_path)
with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
handle.write("1")
- createrepo_unsafe(path, None)
+ createrepo_unsafe(path)
if add_comps:
assert "--groupfile" in mc_run_cmd_unsafe.call_args[0][0]
else:
@@ -250,7 +216,7 @@ class TestCreaterepo(object):
with open(os.path.join(repo_path, "repomd.xml"), "w") as handle:
handle.write("1")
- createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel")
+ createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
assert mc_run_cmd_unsafe.call_args[0][0] == expected
def test_createrepo_devel_generated_commands(self, mc_run_cmd_unsafe):
@@ -266,7 +232,7 @@ class TestCreaterepo(object):
for path, expected in [(path_epel_5, expected_epel_5), (path_fedora, expected_fedora)]:
os.makedirs(path)
- createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel")
+ createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
assert os.path.exists(os.path.join(path, "devel"))
assert mc_run_cmd_unsafe.call_args[0][0] == expected
# assert mc_popen.call_args == mock.call(expected, stderr=-1, stdout=-1)
@@ -278,5 +244,5 @@ class TestCreaterepo(object):
for path in [path_epel_5, path_fedora]:
os.makedirs(path)
- createrepo_unsafe(path, lock=None, base_url=self.base_url, dest_dir="devel")
+ createrepo_unsafe(path, base_url=self.base_url, dest_dir="devel")
assert os.path.exists(os.path.join(path, "devel"))
8 years, 8 months
[copr] master: [backend] split backend daemon: extracting RedisLogHandler (54ea481)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 54ea481e3bf4b8973cefa1d1007ba698d27ed786
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Sep 11 15:01:04 2015 +0200
[backend] split backend daemon: extracting RedisLogHandler
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 57 --------------------
backend/backend/daemons/log.py | 4 +-
backend/run/copr_run_logger.py | 20 +++++++
...ackend-vmm.service => copr-backend-log.service} | 5 +-
backend/run/systemd/copr-backend-vmm.service | 2 +-
backend/run/systemd/copr-backend.service | 1 +
6 files changed, 27 insertions(+), 62 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 1dd1385..4a0096b 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -24,19 +24,9 @@ from backend.frontend import FrontendClient
from ..exceptions import CoprBackendError
from ..helpers import BackendConfigReader, get_redis_logger
from .job_grab import CoprJobGrab
-from .log import RedisLogHandler
from .dispatcher import Worker
-from .vm_master import VmMaster
-
-from ..vm_manage.event_handle import EventHandler
-from ..vm_manage.manager import VmManager
-from ..vm_manage.spawn import Spawner
-from ..vm_manage.check import HealthChecker
-from ..vm_manage.terminate import Terminator
-
-
class CoprBackend(object):
"""
@@ -64,11 +54,8 @@ class CoprBackend(object):
self.update_conf()
self.lock = multiprocessing.Lock()
-
self.task_queues = {}
- # event format is a dict {when:time, who:[worker|logger|job|main],
- # what:str}
self.frontend_client = FrontendClient(self.opts)
self.is_running = False
@@ -102,11 +89,6 @@ class CoprBackend(object):
self.clean_task_queues()
- def _start_log_handler(self):
- self.redis_log_handler = RedisLogHandler(self.opts)
- self.redis_log_handler.start()
- time.sleep(1)
-
def _start_job_grab(self):
self.log.info("Starting up Job Grabber")
self._jobgrab = CoprJobGrab(opts=self.opts,
@@ -116,42 +98,13 @@ class CoprBackend(object):
def init_sub_process(self):
"""
- - Create backend logger
- Create job grabber
- - Create VM management
"""
-
- self._start_log_handler()
self._start_job_grab()
- # self._start_vmm()
-
- def _start_vmm(self):
- self.log.info("Creating VM Spawner, HealthChecker, Terminator")
- self.spawner = Spawner(self.opts)
- self.checker = HealthChecker(self.opts)
- self.terminator = Terminator(self.opts)
- self.vm_manager = VmManager(opts=self.opts,
- logger=self.log,
- )
- self.vm_manager.post_init()
- self.log.info("Starting up VM EventHandler")
- self.event_handler = EventHandler(self.opts,
- vmm=self.vm_manager,
- terminator=self.terminator)
- self.event_handler.post_init()
- self.event_handler.start()
-
- self.log.info("Starting up VM Master")
- self.vm_master = VmMaster(self.opts,
- vmm=self.vm_manager,
- spawner=self.spawner,
- checker=self.checker)
- self.vm_master.start()
def ensure_sub_processes_alive(self):
if self.is_running:
for proc, start_method in [
- (self.redis_log_handler, self._start_log_handler),
(self._jobgrab, self._start_job_grab),
]:
@@ -244,16 +197,6 @@ class CoprBackend(object):
self.log.exception(err)
return
- try:
- self.vm_master.terminate()
- except Exception:
- self.log.exception("Failed to terminate vmm daemon")
- try:
- self.event_handler.terminate()
- self.event_handler.join()
- except Exception:
- self.log.exception("Failed to terminate vm EventHandler")
-
def run(self):
"""
Starts backend process. Control sub process start/stop.
diff --git a/backend/backend/daemons/log.py b/backend/backend/daemons/log.py
index f74962e..01c0ca0 100644
--- a/backend/backend/daemons/log.py
+++ b/backend/backend/daemons/log.py
@@ -62,14 +62,14 @@ class CustomFilter(logging.Filter):
return True
-class RedisLogHandler(Process):
+class RedisLogHandler(object):
"""
Single point to collect logs through redis pub/sub and write
them through standard python logging lib
"""
def __init__(self, opts):
- Process.__init__(self, name="log_handler")
+ # Process.__init__(self, name="log_handler")
self.opts = opts
diff --git a/backend/run/copr_run_logger.py b/backend/run/copr_run_logger.py
new file mode 100755
index 0000000..6b41a74
--- /dev/null
+++ b/backend/run/copr_run_logger.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python2
+# coding: utf-8
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
+from backend.helpers import get_backend_opts
+from backend.daemons.log import RedisLogHandler
+
+
+def main():
+ opts = get_backend_opts()
+ handler = RedisLogHandler(opts)
+ handler.run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/run/systemd/copr-backend-vmm.service b/backend/run/systemd/copr-backend-log.service
similarity index 59%
copy from backend/run/systemd/copr-backend-vmm.service
copy to backend/run/systemd/copr-backend-log.service
index 9784b93..1577743 100644
--- a/backend/run/systemd/copr-backend-vmm.service
+++ b/backend/run/systemd/copr-backend-log.service
@@ -1,13 +1,14 @@
[Unit]
-Description=Copr Backend service
+Description=Copr Backend service, Log Handler component
After=syslog.target network.target auditd.service
+Before=copr-backend.service
[Service]
Type=simple
Environment="PYTHONPATH=/usr/share/copr/"
User=copr
Group=copr
-ExecStart=/usr/bin/copr_run_vmm.py
+ExecStart=/usr/bin/copr_run_logger.py
[Install]
WantedBy=multi-user.target
diff --git a/backend/run/systemd/copr-backend-vmm.service b/backend/run/systemd/copr-backend-vmm.service
index 9784b93..40a3767 100644
--- a/backend/run/systemd/copr-backend-vmm.service
+++ b/backend/run/systemd/copr-backend-vmm.service
@@ -1,5 +1,5 @@
[Unit]
-Description=Copr Backend service
+Description=Copr Backend service, Virtual Machine Management component
After=syslog.target network.target auditd.service
[Service]
diff --git a/backend/run/systemd/copr-backend.service b/backend/run/systemd/copr-backend.service
index 8aa36db..ea43924 100644
--- a/backend/run/systemd/copr-backend.service
+++ b/backend/run/systemd/copr-backend.service
@@ -2,6 +2,7 @@
Description=Copr Backend service, Virtual Machine Management component
After=syslog.target network.target auditd.service
Requires=copr-backend-vmm.service
+Wants=copr-backend-log.service
[Service]
Type=simple
8 years, 8 months
[copr] master: [backend] split backend daemon: extracting VMM (fc25175)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit fc251751427ab7d11f3528cd9ca04757b740a76f
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Sep 11 13:57:48 2015 +0200
[backend] split backend daemon: extracting VMM
>---------------------------------------------------------------
backend/backend/daemons/backend.py | 2 +-
backend/backend/helpers.py | 18 ++++++
backend/copr-backend.spec | 5 +-
backend/run/copr_run_vmm.py | 62 ++++++++++++++++++++
.../systemd/copr-backend-vmm.service} | 4 +-
backend/{ => run/systemd}/copr-backend.service | 3 +-
backend/tests/deamons/test_backend.py | 2 -
7 files changed, 87 insertions(+), 9 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index a78791d..1dd1385 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -123,7 +123,7 @@ class CoprBackend(object):
self._start_log_handler()
self._start_job_grab()
- self._start_vmm()
+ # self._start_vmm()
def _start_vmm(self):
self.log.info("Creating VM Spawner, HealthChecker, Terminator")
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 0bcf97f..e26d5f7 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -367,3 +367,21 @@ def silent_remove(filename):
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
raise # re-raise exception if a different error occured
+
+
+def get_backend_opts():
+ args = sys.argv[1:]
+ parser = optparse.OptionParser("\ncopr-be [options]")
+ parser.add_option("-c", "--config", default="/etc/copr/copr-be.conf",
+ dest="config_file",
+ help="config file to use for copr-be run")
+
+ opts, args = parser.parse_args(args)
+ if not os.path.exists(opts.config_file):
+ sys.stderr.write("No config file found at: {0}\n".format(
+ opts.config_file))
+ sys.exit(1)
+
+ config_file = os.path.abspath(opts.config_file)
+ config_reader = BackendConfigReader(config_file, {})
+ return config_reader.read()
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index 3cea572..3ddd257 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -159,7 +159,8 @@ touch %{buildroot}%{_var}/log/copr/prune_old.log
touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid
-install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
+#install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
+cp -a run/systemd/*.service %{buildroot}/%{_unitdir}/
install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr
@@ -225,7 +226,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%doc %{_pkgdocdir}/playbooks
%dir %{_sysconfdir}/copr
%config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-be.conf
-%{_unitdir}/copr-backend.service
+%{_unitdir}/*.service
%{_tmpfilesdir}/copr-backend.conf
%{_bindir}/*
diff --git a/backend/run/copr_run_vmm.py b/backend/run/copr_run_vmm.py
new file mode 100755
index 0000000..f3c03f0
--- /dev/null
+++ b/backend/run/copr_run_vmm.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python2
+# coding: utf-8
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+
+from setproctitle import setproctitle
+
+from backend.vm_manage.manager import VmManager
+from backend.vm_manage.spawn import Spawner
+from backend.vm_manage.event_handle import EventHandler
+from backend.vm_manage.terminate import Terminator
+from backend.vm_manage.check import HealthChecker
+from backend.daemons.vm_master import VmMaster
+from backend.helpers import get_redis_logger, get_backend_opts
+
+
+class VmmRunner(object):
+
+ def __init__(self, opts):
+ self.opts = opts
+ self.log = get_redis_logger(self.opts, "vmm.main", "vmm")
+
+ def run(self):
+ # todo: 1) do all ansible calls through subprocess
+ # 2) move to Python 3 and asyncIO all in one thread + executors
+ # ... -> eliminate multiprocessing here,
+ # ... possible to use simple logging, with redis handler
+
+ self.log.info("Creating VM Spawner, HealthChecker, Terminator")
+ self.spawner = Spawner(self.opts)
+ self.checker = HealthChecker(self.opts)
+ self.terminator = Terminator(self.opts)
+ self.vm_manager = VmManager(
+ opts=self.opts, logger=self.log,
+ )
+ self.vm_manager.post_init()
+ self.log.info("Starting up VM EventHandler")
+ self.event_handler = EventHandler(self.opts,
+ vmm=self.vm_manager,
+ terminator=self.terminator)
+ self.event_handler.post_init()
+ self.event_handler.start()
+
+ self.log.info("Starting up VM Master")
+ self.vm_master = VmMaster(self.opts,
+ vmm=self.vm_manager,
+ spawner=self.spawner,
+ checker=self.checker)
+ self.vm_master.start()
+ setproctitle("Copr VMM base process")
+
+
+def main():
+ opts = get_backend_opts()
+ vr = VmmRunner(opts)
+ vr.run()
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/copr-backend.service b/backend/run/systemd/copr-backend-vmm.service
similarity index 66%
copy from backend/copr-backend.service
copy to backend/run/systemd/copr-backend-vmm.service
index 71fa067..9784b93 100644
--- a/backend/copr-backend.service
+++ b/backend/run/systemd/copr-backend-vmm.service
@@ -4,12 +4,10 @@ After=syslog.target network.target auditd.service
[Service]
Type=simple
-PIDFile=/var/run/copr-backend/copr-be.pid
Environment="PYTHONPATH=/usr/share/copr/"
User=copr
Group=copr
-ExecStart=/usr/bin/copr_be.py
-ExecReload=/bin/kill -HUP $MAINPID
+ExecStart=/usr/bin/copr_run_vmm.py
[Install]
WantedBy=multi-user.target
diff --git a/backend/copr-backend.service b/backend/run/systemd/copr-backend.service
similarity index 73%
copy from backend/copr-backend.service
copy to backend/run/systemd/copr-backend.service
index 71fa067..8aa36db 100644
--- a/backend/copr-backend.service
+++ b/backend/run/systemd/copr-backend.service
@@ -1,6 +1,7 @@
[Unit]
-Description=Copr Backend service
+Description=Copr Backend service, Virtual Machine Management component
After=syslog.target network.target auditd.service
+Requires=copr-backend-vmm.service
[Service]
Type=simple
diff --git a/backend/tests/deamons/test_backend.py b/backend/tests/deamons/test_backend.py
index b3da64d..37fade0 100644
--- a/backend/tests/deamons/test_backend.py
+++ b/backend/tests/deamons/test_backend.py
@@ -119,13 +119,11 @@ class TestBackend(object):
)
-
@pytest.fixture
def init_be(self):
self.be = CoprBackend(self.config_file, self.ext_opts)
self.be.log = MagicMock()
-
@pytest.yield_fixture
def mc_vmm_stuff(self):
patchers = []
8 years, 8 months
[copr] master: [backend] fixed tests (14e17f5)
by vgologuz＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 14e17f586eceef53b5149e3ed2f170eafa76ded3
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Sep 10 17:46:44 2015 +0200
[backend] fixed tests
>---------------------------------------------------------------
backend/tests/deamons/test_backend.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend/tests/deamons/test_backend.py b/backend/tests/deamons/test_backend.py
index 01e61f5..b3da64d 100644
--- a/backend/tests/deamons/test_backend.py
+++ b/backend/tests/deamons/test_backend.py
@@ -56,8 +56,6 @@ def mc_daemon_context():
yield obj
-
-
class TestBackend(object):
def setup_method(self, method):
@@ -125,12 +123,14 @@ class TestBackend(object):
@pytest.fixture
def init_be(self):
self.be = CoprBackend(self.config_file, self.ext_opts)
+ self.be.log = MagicMock()
@pytest.yield_fixture
def mc_vmm_stuff(self):
patchers = []
- for klass_name in ["Spawner", "HealthChecker", "Terminator", "VmManager", "VmMaster"]:
+ for klass_name in ["Spawner", "HealthChecker", "Terminator",
+ "VmManager", "VmMaster", "EventHandler"]:
patcher = mock.patch("{}.{}".format(MODULE_REF, klass_name))
patchers.append(patcher)
setattr(self, klass_name, patcher.start())
8 years, 8 months
[copr] master: [doc][api 2] review project_chroot resource (6a7becd)
by asamalik＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6a7becde3e970174242a30d55be76746f5c6a72a
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Sep 10 16:13:25 2015 +0200
[doc][api 2] review project_chroot resource
>---------------------------------------------------------------
.../docs/api_2/source/Resources/project_chroot.rst | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/frontend/docs/api_2/source/Resources/project_chroot.rst b/frontend/docs/api_2/source/Resources/project_chroot.rst
index 48288dc..a35c8bf 100644
--- a/frontend/docs/api_2/source/Resources/project_chroot.rst
+++ b/frontend/docs/api_2/source/Resources/project_chroot.rst
@@ -91,7 +91,7 @@ Enable chroot for project
**REQUIRE AUTH**
Enables chroot for the Copr project.
- Available `chroot` names could be obtained from :doc:`./mock_chroot` resource.
+ Available `chroot` names can be obtained from :doc:`./mock_chroot` resource.
:param int project_id: a unique identifier of the Copr project.
8 years, 8 months
[copr] master: [API 2] add build submitter (a3a7619)
by asamalik＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit a3a7619dd581eb0869f9920a03993616d8271937
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Sep 10 17:13:02 2015 +0200
[API 2] add build submitter
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/models.py | 4 ++++
frontend/coprs_frontend/coprs/rest_api/schemas.py | 2 ++
2 files changed, 6 insertions(+), 0 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index e52087c..63ef2bd 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -370,6 +370,10 @@ class Build(db.Model, helpers.Serializer):
chroots = association_proxy("build_chroots", "mock_chroot")
@property
+ def user_name(self):
+ return self.user.name
+
+ @property
def fail_type_text(self):
return helpers.FailTypeEnum(self.fail_type)
diff --git a/frontend/coprs_frontend/coprs/rest_api/schemas.py b/frontend/coprs_frontend/coprs/rest_api/schemas.py
index b28d1c6..2144b54 100644
--- a/frontend/coprs_frontend/coprs/rest_api/schemas.py
+++ b/frontend/coprs_frontend/coprs/rest_api/schemas.py
@@ -145,6 +145,8 @@ class BuildSchema(Schema):
id = fields.Int(dump_only=True)
state = fields.Str()
+ submitter = fields.Str(dump_only=True, attribute="user_name")
+
built_packages = BuiltPackages(dump_only=True)
package_version = fields.Str(dump_only=True, attribute="pkg_version")
package_name = fields.Str(dump_only=True)
8 years, 8 months
[copr] master: [doc][api 2] review build_task (15e5343)
by asamalik＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 15e53431b7df4685e9927236f8e79553ad565e9b
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Sep 10 17:11:08 2015 +0200
[doc][api 2] review build_task
>---------------------------------------------------------------
.../docs/api_2/source/Resources/build_task.rst | 8 ++++----
1 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/frontend/docs/api_2/source/Resources/build_task.rst b/frontend/docs/api_2/source/Resources/build_task.rst
index 2212475..bca8d4c 100644
--- a/frontend/docs/api_2/source/Resources/build_task.rst
+++ b/frontend/docs/api_2/source/Resources/build_task.rst
@@ -34,7 +34,7 @@ result_dir_url str(URL) location of the build results
.. note::
- Build Chroot doesn't currently support any modifications,
+ Build Task doesn't currently support any modifications,
so all fields are read-only.
List build tasks
@@ -51,14 +51,14 @@ List build tasks
:query int build_id:
select build tasks from one project,
when used query parameters ``owner`` and ``project_id`` are ignored
-
- :query int offset: offset number, default value is 0
- :query int limit: limit number, default value is 100
:query str state:
select builds in particular state, allowed values:
``failed``, ``succeeded``, ``canceled``, ``running``,
``pending``, ``starting``, ``importing``
+ :query int offset: offset number, default value is 0
+ :query int limit: limit number, default value is 100
+
:statuscode 200: no error
:statuscode 404: build not found
8 years, 8 months
[copr] master: [doc][api 2] review project resource (bcb8559)
by asamalik＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit bcb855959e915482f0df2a9c3de7f2f25d3219c9
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Sep 10 14:57:45 2015 +0200
[doc][api 2] review project resource
>---------------------------------------------------------------
frontend/docs/api_2/source/Resources/project.rst | 14 +++++++-------
1 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/frontend/docs/api_2/source/Resources/project.rst b/frontend/docs/api_2/source/Resources/project.rst
index aaad18f..ce9a49b 100644
--- a/frontend/docs/api_2/source/Resources/project.rst
+++ b/frontend/docs/api_2/source/Resources/project.rst
@@ -33,11 +33,11 @@ Project fields
Field Type Can edit? Description
================== ==================== ========= ===============
id number no unique identifier
-owner string no login of the project owner
+owner string no username of the project owner
name string no name of the project, MUST be specified during a project creation
description string yes project description
instructions string yes installation instructions
-homepage string(URL) yes project homepage
+homepage string(URL) yes project homepage URL
contact string(URL or email) yes contact with the project maintainer
disable_createrepo bool yes disables automatic repository metadata generation
build_enable_net bool yes set default value for new builds option `enable_net`
@@ -105,7 +105,7 @@ Create new project
Creates new Copr project.
- Additionally to described before `Project fields`_ the user could specify field `chroots` which contains list of chroots to be enabled.
+ Additionally to what is described before in `Project fields`_, the user could specify a field `chroots`, which contains a list of chroots to be enabled.
Available `chroot` names could be obtained from :doc:`./mock_chroot`
:resheader Location: contains URL to the newly created project entity
@@ -147,12 +147,12 @@ Get project details
-------------------
.. http:get:: /api_2/projects/(int:project_id)
- Returns details about Copr project
+ Returns details about given Copr project
:param int project_id: a unique identifier of the Copr project.
- :query bool show_builds: embed :doc:`./build` entities owned by this project into the result, default is False
- :query bool show_chroots: embed :doc:`./project_chroot` sub-resources into the result, default is False
+ :query bool show_builds: include :doc:`./build` entities owned by this project in the result, default is False
+ :query bool show_chroots: include :doc:`./project_chroot` sub-resources in the result, default is False
:statuscode 200: no error
:statuscode 404: project not found
@@ -267,7 +267,7 @@ Modify project
Updates Copr project.
- .. note:: You couldn't enabled or disable project chroots here, use :doc:`./project_chroot` resource.
+ .. note:: Project chroots can not be enabled or disabled here. Use :doc:`./project_chroot` resource for this purpose.
:param project_id: a unique identifier of the Copr project.
8 years, 8 months
[copr] master: [doc][api 2] build review + add submitter to the API (64cf275)
by asamalik＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 64cf2750b271d96762c65920a91d5759e370e10a
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Sep 10 17:04:38 2015 +0200
[doc][api 2] build review + add submitter to the API
>---------------------------------------------------------------
frontend/docs/api_2/source/Resources/build.rst | 87 ++++++++++++++++------
frontend/docs/api_2/source/Resources/project.rst | 6 +-
2 files changed, 70 insertions(+), 23 deletions(-)
diff --git a/frontend/docs/api_2/source/Resources/build.rst b/frontend/docs/api_2/source/Resources/build.rst
index f90b5ef..66c85ac 100644
--- a/frontend/docs/api_2/source/Resources/build.rst
+++ b/frontend/docs/api_2/source/Resources/build.rst
@@ -1,10 +1,10 @@
Build
=====
-Build resources allows to submit new builds and access current build progress.
+Build resource allows to submit new builds and access current build progress.
-Build in fact consists of the few tasks, one per chroot, and detail information
-is available through sub-resource :doc:`./build_task`.
+In fact, build consists of a few tasks, one per chroot, and detailed information
+is available through resource :doc:`./build_task`.
@@ -37,7 +37,8 @@ Structure of the build entity
"ended_on": 1440754058,
"package_version": "2.0.0b5-1.fc22",
"package_name": "python-marshmallow",
- "id": 106882
+ "id": 106882,
+ "submitter": "asamalik"
}
@@ -58,19 +59,20 @@ package_name string name of the source package
enable_net bool defines if network is available during the build
repos list of string list of additional repositories enabled during the build
built_packages list of hash maps list of the built packages, each hash map has two keys: ``name`` and ``version``
+submitter string name of the user who submitted the build
================== ==================== ===============
.. note::
Only the ``state`` field is editable by the PUT method.
- All other fields are read only.
- There is a different structure used for build creation, see details at `Submit new build`_
+ All other fields are read-only.
+ There is a different structure used for build creation, see details at `Submit new build`_.
List builds
-----------
.. http:post:: /api_2/builds
- Return a list of build according to the given query parameters.
+ Returns a list of builds according to the given query parameters.
:query str owner: select only builds from projects owned by this user
:query str project_id: select only projects owned by this project
@@ -131,7 +133,8 @@ List builds
"state": "succeeded",
"ended_on": 1441366969,
"id": 106897,
- "repos": []
+ "repos": [],
+ "submitter": "asamalik"
}
}
]
@@ -140,11 +143,11 @@ List builds
Submit new build
----------------
-**REQUIRE AUTH**
+**REQUIRES AUTH**
-Allows to submit new build. Copr services currently provides the following options for build submission:
+There are more ways to submit new build. Copr services currently provides the following options for build submission:
-From srpm url
+From SRPM URL
~~~~~~~~~~~~~
.. code-block:: javascript
@@ -159,8 +162,8 @@ From srpm url
Field Type Description
================== ==================== ===============
project_id int identifier of the parent project
- chroots list of strings what chroots should be used for build
- srpm_url string(URL) url to the publicly available source package
+ chroots list of strings which chroots should be used for build
+ srpm_url string(URL) URL to the publicly available source package
enable_net bool allows to disable network access during the build, default: True
================== ==================== ===============
@@ -197,14 +200,14 @@ From srpm url
HTTP/1.1 201 CREATED
Location: /api_2/builds/106897
-Using file upload
-~~~~~~~~~~~~~~~~~
+Using SRPM file upload
+~~~~~~~~~~~~~~~~~~~~~~
To upload source package you MUST use ``multipart/form-data`` content type.
-Addition build information MUST present in ``metadata`` part in JSON format. Source package
+An additional build information MUST be present in ``metadata`` part in JSON format. Source package
MUST be uploaded as binary ``srpm`` file.
- **Build info**
+ ****
.. code-block:: javascript
@@ -226,8 +229,8 @@ MUST be uploaded as binary ``srpm`` file.
.. http:post:: /api_2/builds
:reqheader Content-Type: MUST be a ``multipart/form-data``
- :formparam metadata: JSON with the build info
- :formparam srpm: file with source package
+ :formparam metadata: JSON with the build info, MUST have a content type ``application/json``
+ :formparam srpm: file with source package, MUST have a content type ``application/x-rpm``
:resheader Location: contains URL to the created build
@@ -235,7 +238,46 @@ MUST be uploaded as binary ``srpm`` file.
:statuscode 400: user data doesn't satisfy some requirements
:statuscode 403: authorization failed
- **Example**
+ .. note:: Using a ``multipart/form-data`` might not be nice to read. To make your life a bit brighter, a Python example is included below.
+
+ **Example request**:
+
+ .. sourcecode:: http
+
+ POST /api_2/builds HTTP/1.1
+ Host: copr.fedoraproject.org
+ Authorization: Basic base64=encoded=string
+ Content-Length: xxxx
+ Content-Type: multipart/form-data; boundary=--------------------31063722920652
+
+ ------------------------------31063722920652
+ Content-Disposition: form-data; name="metadata"
+ Content-Type: application/json
+
+ {
+ "project_id": 3985,
+ "chroots": ["fedora-22-i386", "fedora-21-i386"],
+ "enable_net": false
+ }
+
+ ------------------------------31063722920652
+ Content-Disposition: form-data; name="srpm"; filename="package-2.6-fc21.src.rpm"
+ Content-Type: application/x-rpm
+
+ << SRPM BINARY CONTENT HERE >>
+
+ -----------------------------31063722920652--
+
+
+ **Response**:
+
+ .. sourcecode:: http
+
+ HTTP/1.1 201 CREATED
+ Location: /api_2/builds/106897
+
+
+ **Python Example**:
Here we use python-requests_ lib:
@@ -340,7 +382,8 @@ Get build details
"state": "succeeded",
"ended_on": 1441366969,
"id": 106897,
- "repos": []
+ "repos": [],
+ "submitter": "asamalik"
}
}
@@ -384,7 +427,7 @@ Delete build
**REQUIRE AUTH**
- Deletes build and schedules deletion of build result at Copr backend
+ Deletes build and schedules deletion of build result from the Copr backend
:param int build_id: a unique identifier of the build
diff --git a/frontend/docs/api_2/source/Resources/project.rst b/frontend/docs/api_2/source/Resources/project.rst
index ce9a49b..71324c5 100644
--- a/frontend/docs/api_2/source/Resources/project.rst
+++ b/frontend/docs/api_2/source/Resources/project.rst
@@ -202,6 +202,9 @@ Get project details
"_links": { },
"build": {
"enable_net": true,
+ "source_metadata": {
+ "url": "http://miroslav.suchy.cz/copr/copr-ping-1-1.fc20.src.rpm"
+ },
"submitted_on": 1422379448,
"repos": [],
"results": "https://copr-be.cloud.fedoraproject.org/results/jmiahman/Synergy-Linux/",
@@ -212,7 +215,8 @@ Get project details
"ended_on": 1422379584,
"timeout": 21600,
"pkg_version": "21.6-1.fc21",
- "id": 69493
+ "id": 69493,
+ "submitter": "asamalik"
}
},
{ }
8 years, 8 months
[copr] master: add f23 to releaser (042d415)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 042d415fb6f339b929a3dda8792c1caeea5c1d79
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Thu Sep 10 16:23:01 2015 +0200
add f23 to releaser
>---------------------------------------------------------------
rel-eng/releasers.conf | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/rel-eng/releasers.conf b/rel-eng/releasers.conf
index 7a8b454..d298a14 100644
--- a/rel-eng/releasers.conf
+++ b/rel-eng/releasers.conf
@@ -12,7 +12,7 @@ branches = f23
[fedora-git-all]
releaser = tito.release.FedoraGitReleaser
-branches = master f22 f21 el6 epel7
+branches = master f23 f22 f21 el6 epel7
[copr-msuchy]
releaser = tito.release.CoprReleaser
8 years, 8 months
