Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 4b9286a614078b8656d5f58ff3711e6204528c00
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Sep 11 20:12:47 2015 +0200
[backend] split backend daemon: extracting JobGrab
---------------------------------------------------------------
backend/backend/daemons/backend.py | 36 +-------------------
backend/backend/daemons/dispatcher.py | 7 +---
backend/backend/daemons/job_grab.py | 6 +--
backend/backend/mockremote/__init__.py | 10 +-----
.../{copr_run_logger.py => copr_run_job_grab.py} | 8 +++--
...nd-log.service => copr-backend-jobgrab.service} | 6 ++--
backend/run/systemd/copr-backend-vmm.service | 1 +
backend/run/systemd/copr-backend.service | 3 +-
8 files changed, 16 insertions(+), 61 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 4a0096b..5a89f82 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -23,7 +23,6 @@ from backend.frontend import FrontendClient
from ..exceptions import CoprBackendError
from ..helpers import BackendConfigReader, get_redis_logger
-from .job_grab import CoprJobGrab
from .dispatcher import Worker
@@ -53,7 +52,6 @@ class CoprBackend(object):
self.opts = None
self.update_conf()
- self.lock = multiprocessing.Lock()
self.task_queues = {}
self.frontend_client = FrontendClient(self.opts)
@@ -89,31 +87,6 @@ class CoprBackend(object):
self.clean_task_queues()
- def _start_job_grab(self):
- self.log.info("Starting up Job Grabber")
- self._jobgrab = CoprJobGrab(opts=self.opts,
- frontend_client=self.frontend_client,
- lock=self.lock)
- self._jobgrab.start()
-
- def init_sub_process(self):
- """
- - Create job grabber
- """
- self._start_job_grab()
-
- def ensure_sub_processes_alive(self):
- if self.is_running:
- for proc, start_method in [
- (self._jobgrab, self._start_job_grab),
-
- ]:
- if not proc.is_alive():
- self.log.error("Process `{}` died unexpectedly,
restarting".format(proc))
- proc.terminate()
- proc.join()
- start_method()
-
def update_conf(self):
"""
Update backend config from config file
@@ -142,8 +115,7 @@ class CoprBackend(object):
opts=self.opts,
frontend_client=self.frontend_client,
worker_num=self.max_worker_num_by_group_id[group_id],
- group_id=group_id,
- lock=self.lock
+ group_id=group_id
)
self.workers_by_group_id[group_id].append(w)
@@ -202,13 +174,9 @@ class CoprBackend(object):
Starts backend process. Control sub process start/stop.
"""
self.update_conf()
-
self.init_task_queues()
- self.init_sub_process()
time.sleep(1)
-
self.log.info("Initial config: {}".format(self.opts))
- self.log.info("Sub processes was started")
try:
self.log.info("Rescheduling old unfinished builds")
@@ -222,8 +190,6 @@ class CoprBackend(object):
# re-read config into opts
self.update_conf()
- self.ensure_sub_processes_alive()
-
for group in self.opts.build_groups:
group_id = group["id"]
diff --git a/backend/backend/daemons/dispatcher.py
b/backend/backend/daemons/dispatcher.py
index 3b379b2..023e929 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -39,11 +39,10 @@ class Worker(multiprocessing.Process):
:param Munch opts: backend config
:param int worker_num: worker number
:param int group_id: group_id from the set of groups defined in config
- :param lock: (:py:class:`multiprocessing.Lock`) global backend lock
"""
- def __init__(self, opts, frontend_client, worker_num, group_id, lock=None):
+ def __init__(self, opts, frontend_client, worker_num, group_id):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
@@ -60,7 +59,6 @@ class Worker(multiprocessing.Process):
# event queue for communicating back to dispatcher
self.kill_received = False
- self.lock = lock
self.frontend_client = frontend_client
self.vm_name = None
@@ -299,8 +297,7 @@ class Worker(multiprocessing.Process):
builder_host=self.vm_ip,
job=job,
logger=build_logger,
- opts=self.opts,
- lock=self.lock,
+ opts=self.opts
)
mr.check()
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index df6e98e..7a15b97 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -29,7 +29,7 @@ from ..frontend import FrontendClient
from ..vm_manage.manager import VmManager
-class CoprJobGrab(Process):
+class CoprJobGrab(object):
"""
Fetch jobs from the Frontend
@@ -44,9 +44,8 @@ class CoprJobGrab(Process):
"""
- def __init__(self, opts, frontend_client, lock):
+ def __init__(self, opts, frontend_client):
# base class initialization
- super(CoprJobGrab, self).__init__(name="jobgrab")
self.opts = opts
self.arch_to_group_id_map = dict()
@@ -58,7 +57,6 @@ class CoprJobGrab(Process):
self.task_queues_by_group = {}
self.added_jobs_dict = dict() # task_id -> task dict
- self.lock = lock # used only for createrepo calls
self.frontend_client = frontend_client
diff --git a/backend/backend/mockremote/__init__.py
b/backend/backend/mockremote/__init__.py
index ceffc28..ece50b7 100755
--- a/backend/backend/mockremote/__init__.py
+++ b/backend/backend/mockremote/__init__.py
@@ -80,7 +80,7 @@ class MockRemote(object):
# idea: send events according to the build progress to handler
def __init__(self, builder_host, job, logger,
- repos=None, opts=None, lock=None,):
+ repos=None, opts=None):
"""
:param builder_host: builder hostname or ip
@@ -101,8 +101,6 @@ class MockRemote(object):
:param macros: { "copr_username": ...,
"copr_projectname": ...,
"vendor": ...}
- :param multiprocessing.Lock lock: instance of Lock shared between
- Copr backend process
:param Munch opts: builder options, used keys::
:ivar build_user: user to run as/connect as on builder systems
@@ -132,12 +130,6 @@ class MockRemote(object):
self.log = logger
self.job = job
- # TODO: remove or re-implement
- # self.cont = cont # unused since we build only one pkg at time
- # self.recurse = recurse
-
- self.lock = lock
-
self.log.info("Setting up builder: {0}".format(builder_host))
# TODO: add option "builder_log_level" to backend config
self.log.setLevel(logging.INFO)
diff --git a/backend/run/copr_run_logger.py b/backend/run/copr_run_job_grab.py
old mode 100755
new mode 100644
similarity index 62%
copy from backend/run/copr_run_logger.py
copy to backend/run/copr_run_job_grab.py
index 6b41a74..47a549b
--- a/backend/run/copr_run_logger.py
+++ b/backend/run/copr_run_job_grab.py
@@ -7,13 +7,15 @@ from __future__ import division
from __future__ import absolute_import
from backend.helpers import get_backend_opts
-from backend.daemons.log import RedisLogHandler
+from backend.daemons.job_grab import CoprJobGrab
+from backend.frontend import FrontendClient
def main():
opts = get_backend_opts()
- handler = RedisLogHandler(opts)
- handler.run()
+ fc = FrontendClient(opts)
+ grabber = CoprJobGrab(opts, frontend_client=fc)
+ grabber.run()
if __name__ == "__main__":
diff --git a/backend/run/systemd/copr-backend-log.service
b/backend/run/systemd/copr-backend-jobgrab.service
similarity index 59%
copy from backend/run/systemd/copr-backend-log.service
copy to backend/run/systemd/copr-backend-jobgrab.service
index 1577743..e2cef79 100644
--- a/backend/run/systemd/copr-backend-log.service
+++ b/backend/run/systemd/copr-backend-jobgrab.service
@@ -1,14 +1,14 @@
[Unit]
-Description=Copr Backend service, Log Handler component
+Description=Copr Backend service, Job Grabber component
After=syslog.target network.target auditd.service
-Before=copr-backend.service
+After=copr-backend.service
[Service]
Type=simple
Environment="PYTHONPATH=/usr/share/copr/"
User=copr
Group=copr
-ExecStart=/usr/bin/copr_run_logger.py
+ExecStart=/usr/bin/copr_run_job_grab.py
[Install]
WantedBy=multi-user.target
diff --git a/backend/run/systemd/copr-backend-vmm.service
b/backend/run/systemd/copr-backend-vmm.service
index 40a3767..33b4aed 100644
--- a/backend/run/systemd/copr-backend-vmm.service
+++ b/backend/run/systemd/copr-backend-vmm.service
@@ -1,6 +1,7 @@
[Unit]
Description=Copr Backend service, Virtual Machine Management component
After=syslog.target network.target auditd.service
+Before=copr-backend.service
[Service]
Type=simple
diff --git a/backend/run/systemd/copr-backend.service
b/backend/run/systemd/copr-backend.service
index ce3a4ef..d0a931c 100644
--- a/backend/run/systemd/copr-backend.service
+++ b/backend/run/systemd/copr-backend.service
@@ -1,8 +1,7 @@
[Unit]
Description=Copr Backend service, Workers controller
After=syslog.target network.target auditd.service
-Requires=copr-backend-vmm.service
-Wants=copr-backend-log.service
+Requires=copr-backend-vmm.service copr-backend-jobgrab.service copr-backend-log.service
[Service]
Type=simple