Repository : http://git.fedorahosted.org/cgit/copr.git

On branch : master

commit 4b9286a614078b8656d5f58ff3711e6204528c00 Author: Valentin Gologuzov vgologuz@redhat.com Date: Fri Sep 11 20:12:47 2015 +0200

[backend] split backend daemon: extracting JobGrab

backend/backend/daemons/backend.py | 36 +------------------- backend/backend/daemons/dispatcher.py | 7 +--- backend/backend/daemons/job_grab.py | 6 +-- backend/backend/mockremote/__init__.py | 10 +----- .../{copr_run_logger.py => copr_run_job_grab.py} | 8 +++-- ...nd-log.service => copr-backend-jobgrab.service} | 6 ++-- backend/run/systemd/copr-backend-vmm.service | 1 + backend/run/systemd/copr-backend.service | 3 +- 8 files changed, 16 insertions(+), 61 deletions(-)

diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py index 4a0096b..5a89f82 100644 --- a/backend/backend/daemons/backend.py +++ b/backend/backend/daemons/backend.py @@ -23,7 +23,6 @@ from backend.frontend import FrontendClient

from ..exceptions import CoprBackendError from ..helpers import BackendConfigReader, get_redis_logger -from .job_grab import CoprJobGrab from .dispatcher import Worker

@@ -53,7 +52,6 @@ class CoprBackend(object): self.opts = None self.update_conf()

- self.lock = multiprocessing.Lock() self.task_queues = {}

self.frontend_client = FrontendClient(self.opts) @@ -89,31 +87,6 @@ class CoprBackend(object):

self.clean_task_queues()

- def _start_job_grab(self): - self.log.info("Starting up Job Grabber") - self._jobgrab = CoprJobGrab(opts=self.opts, - frontend_client=self.frontend_client, - lock=self.lock) - self._jobgrab.start() - - def init_sub_process(self): - """ - - Create job grabber - """ - self._start_job_grab() - - def ensure_sub_processes_alive(self): - if self.is_running: - for proc, start_method in [ - (self._jobgrab, self._start_job_grab), - - ]: - if not proc.is_alive(): - self.log.error("Process `{}` died unexpectedly, restarting".format(proc)) - proc.terminate() - proc.join() - start_method() - def update_conf(self): """ Update backend config from config file @@ -142,8 +115,7 @@ class CoprBackend(object): opts=self.opts, frontend_client=self.frontend_client, worker_num=self.max_worker_num_by_group_id[group_id], - group_id=group_id, - lock=self.lock + group_id=group_id )

self.workers_by_group_id[group_id].append(w) @@ -202,13 +174,9 @@ class CoprBackend(object): Starts backend process. Control sub process start/stop. """ self.update_conf() - self.init_task_queues() - self.init_sub_process() time.sleep(1) - self.log.info("Initial config: {}".format(self.opts)) - self.log.info("Sub processes was started")

try: self.log.info("Rescheduling old unfinished builds") @@ -222,8 +190,6 @@ class CoprBackend(object): # re-read config into opts self.update_conf()

- self.ensure_sub_processes_alive() - for group in self.opts.build_groups: group_id = group["id"]

diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index 3b379b2..023e929 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -39,11 +39,10 @@ class Worker(multiprocessing.Process): :param Munch opts: backend config :param int worker_num: worker number :param int group_id: group_id from the set of groups defined in config - :param lock: (:py:class:`multiprocessing.Lock`) global backend lock

"""

- def __init__(self, opts, frontend_client, worker_num, group_id, lock=None): + def __init__(self, opts, frontend_client, worker_num, group_id):

# base class initialization multiprocessing.Process.__init__(self, name="worker-builder") @@ -60,7 +59,6 @@ class Worker(multiprocessing.Process): # event queue for communicating back to dispatcher

self.kill_received = False - self.lock = lock

self.frontend_client = frontend_client self.vm_name = None @@ -299,8 +297,7 @@ class Worker(multiprocessing.Process): builder_host=self.vm_ip, job=job, logger=build_logger, - opts=self.opts, - lock=self.lock, + opts=self.opts ) mr.check()

diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py index df6e98e..7a15b97 100644 --- a/backend/backend/daemons/job_grab.py +++ b/backend/backend/daemons/job_grab.py @@ -29,7 +29,7 @@ from ..frontend import FrontendClient from ..vm_manage.manager import VmManager

-class CoprJobGrab(Process): +class CoprJobGrab(object):

""" Fetch jobs from the Frontend @@ -44,9 +44,8 @@ class CoprJobGrab(Process):

"""

- def __init__(self, opts, frontend_client, lock): + def __init__(self, opts, frontend_client): # base class initialization - super(CoprJobGrab, self).__init__(name="jobgrab")

self.opts = opts self.arch_to_group_id_map = dict() @@ -58,7 +57,6 @@ class CoprJobGrab(Process): self.task_queues_by_group = {}

self.added_jobs_dict = dict() # task_id -> task dict - self.lock = lock # used only for createrepo calls

self.frontend_client = frontend_client

diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index ceffc28..ece50b7 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -80,7 +80,7 @@ class MockRemote(object): # idea: send events according to the build progress to handler

def __init__(self, builder_host, job, logger, - repos=None, opts=None, lock=None,): + repos=None, opts=None):

""" :param builder_host: builder hostname or ip @@ -101,8 +101,6 @@ class MockRemote(object): :param macros: { "copr_username": ..., "copr_projectname": ..., "vendor": ...} - :param multiprocessing.Lock lock: instance of Lock shared between - Copr backend process

:param Munch opts: builder options, used keys:: :ivar build_user: user to run as/connect as on builder systems @@ -132,12 +130,6 @@ class MockRemote(object): self.log = logger self.job = job

- # TODO: remove or re-implement - # self.cont = cont # unused since we build only one pkg at time - # self.recurse = recurse - - self.lock = lock - self.log.info("Setting up builder: {0}".format(builder_host)) # TODO: add option "builder_log_level" to backend config self.log.setLevel(logging.INFO) diff --git a/backend/run/copr_run_logger.py b/backend/run/copr_run_job_grab.py old mode 100755 new mode 100644 similarity index 62% copy from backend/run/copr_run_logger.py copy to backend/run/copr_run_job_grab.py index 6b41a74..47a549b --- a/backend/run/copr_run_logger.py +++ b/backend/run/copr_run_job_grab.py @@ -7,13 +7,15 @@ from __future__ import division from __future__ import absolute_import

from backend.helpers import get_backend_opts -from backend.daemons.log import RedisLogHandler +from backend.daemons.job_grab import CoprJobGrab +from backend.frontend import FrontendClient

def main(): opts = get_backend_opts() - handler = RedisLogHandler(opts) - handler.run() + fc = FrontendClient(opts) + grabber = CoprJobGrab(opts, frontend_client=fc) + grabber.run()

if __name__ == "__main__": diff --git a/backend/run/systemd/copr-backend-log.service b/backend/run/systemd/copr-backend-jobgrab.service similarity index 59% copy from backend/run/systemd/copr-backend-log.service copy to backend/run/systemd/copr-backend-jobgrab.service index 1577743..e2cef79 100644 --- a/backend/run/systemd/copr-backend-log.service +++ b/backend/run/systemd/copr-backend-jobgrab.service @@ -1,14 +1,14 @@ [Unit] -Description=Copr Backend service, Log Handler component +Description=Copr Backend service, Job Grabber component After=syslog.target network.target auditd.service -Before=copr-backend.service +After=copr-backend.service

[Service] Type=simple Environment="PYTHONPATH=/usr/share/copr/" User=copr Group=copr -ExecStart=/usr/bin/copr_run_logger.py +ExecStart=/usr/bin/copr_run_job_grab.py

[Install] WantedBy=multi-user.target diff --git a/backend/run/systemd/copr-backend-vmm.service b/backend/run/systemd/copr-backend-vmm.service index 40a3767..33b4aed 100644 --- a/backend/run/systemd/copr-backend-vmm.service +++ b/backend/run/systemd/copr-backend-vmm.service @@ -1,6 +1,7 @@ [Unit] Description=Copr Backend service, Virtual Machine Management component After=syslog.target network.target auditd.service +Before=copr-backend.service

[Service] Type=simple diff --git a/backend/run/systemd/copr-backend.service b/backend/run/systemd/copr-backend.service index ce3a4ef..d0a931c 100644 --- a/backend/run/systemd/copr-backend.service +++ b/backend/run/systemd/copr-backend.service @@ -1,8 +1,7 @@ [Unit] Description=Copr Backend service, Workers controller After=syslog.target network.target auditd.service -Requires=copr-backend-vmm.service -Wants=copr-backend-log.service +Requires=copr-backend-vmm.service copr-backend-jobgrab.service copr-backend-log.service

[Service] Type=simple