Running Copr on CentOS
by Pavel Raiskup
Is it expected Copr services could (backend/frontend, I am aware dist-git
machine already does) be run on CentOS 7? I was seriously thinking about
it some time ago, I've done some downstream changes, but after some time
those are not useful .. the code is not ready .. and it is getting worse
(hard dependencies on dnf, etc.).
The benefits would be that we wouldn't have to do the updates (and thus
downtimes) so often due to fedora upgrades N -> N+1. I would be ready to
help, but there would have to be consensus that we are not against it.
Pavel
8 years, 2 months
[PATCH] [backend] don't traceback backend if frontend is not up&running
by Pavel Raiskup
---
backend/backend/daemons/backend.py | 11 ++++++-----
backend/backend/daemons/job_grab.py | 7 ++++---
backend/backend/frontend.py | 9 ++++++---
backend/run/copr_run_job_grab.py | 4 +---
4 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 8818f71..934a566 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -52,11 +52,11 @@ class CoprBackend(object):
self.task_queues = {}
- self.frontend_client = FrontendClient(self.opts)
- self.is_running = False
-
self.log = get_redis_logger(self.opts, "backend.main", "backend")
+ self.frontend_client = FrontendClient(self.opts, self.log)
+ self.is_running = False
+
def clean_task_queues(self):
"""
Make sure there is nothing in our task queues
@@ -162,7 +162,7 @@ class CoprBackend(object):
try:
self.log.info("Rescheduling unfinished builds before stop")
- self.frontend_client.reschedule_all_running()
+ self.frontend_client.reschedule_all_running(5)
except RequestException as err:
self.log.exception(err)
return
@@ -178,7 +178,8 @@ class CoprBackend(object):
try:
self.log.info("Rescheduling old unfinished builds")
- self.frontend_client.reschedule_all_running()
+ # 120*5 = 10 minutes
+ self.frontend_client.reschedule_all_running(120)
except RequestException as err:
self.log.exception(err)
return
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index 558ae57..21871ae 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -13,6 +13,8 @@ from requests import get, RequestException
from retask.task import Task
from retask.queue import Queue
+from backend.frontend import FrontendClient
+
from ..actions import Action
from ..constants import JOB_GRAB_TASK_END_PUBSUB
from ..helpers import get_redis_connection, get_redis_logger
@@ -32,12 +34,11 @@ class CoprJobGrab(object):
:param Munch opts: backend config
- :type frontend_client: FrontendClient
:param lock: :py:class:`multiprocessing.Lock` global backend lock
"""
- def __init__(self, opts, frontend_client):
+ def __init__(self, opts):
""" base class initialization """
self.opts = opts
@@ -51,13 +52,13 @@ class CoprJobGrab(object):
self.added_jobs_dict = dict() # task_id -> task dict
- self.frontend_client = frontend_client
self.rc = None
self.channel = None
self.ps_thread = None
self.log = get_redis_logger(self.opts, "backend.job_grab", "job_grab")
+ self.frontend_client = FrontendClient(self.opts, self.log)
def connect_queues(self):
"""
diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py
index 52c91b6..6f442d3 100644
--- a/backend/backend/frontend.py
+++ b/backend/backend/frontend.py
@@ -8,12 +8,13 @@ class FrontendClient(object):
Object to send data back to fronted
"""
- def __init__(self, opts):
+ def __init__(self, opts, logger=None):
super(FrontendClient, self).__init__()
self.frontend_url = "{}/backend".format(opts.frontend_base_url)
self.frontend_auth = opts.frontend_auth
self.msg = None
+ self.log = logger
def _post_to_frontend(self, data, url_path):
"""
@@ -43,6 +44,8 @@ class FrontendClient(object):
"""
for i in range(max_repeats):
try:
+ if i and self.log:
+ self.log.warning("failed to post data to frontend, repeat #{0}".format(i))
return self._post_to_frontend(data, url_path)
except RequestException:
time.sleep(5)
@@ -74,7 +77,7 @@ class FrontendClient(object):
data = {"build_id": build_id, "chroot": chroot_name}
self._post_to_frontend_repeatedly(data, "reschedule_build_chroot")
- def reschedule_all_running(self):
- response = self._post_to_frontend({}, "reschedule_all_running")
+ def reschedule_all_running(self, attempts):
+ response = self._post_to_frontend_repeatedly({}, "reschedule_all_running", attempts)
if response.status_code != 200:
raise RequestException("Failed to reschedule all running jobs")
diff --git a/backend/run/copr_run_job_grab.py b/backend/run/copr_run_job_grab.py
index b39fabe..d1ce730 100755
--- a/backend/run/copr_run_job_grab.py
+++ b/backend/run/copr_run_job_grab.py
@@ -11,13 +11,11 @@ sys.path.append("/usr/share/copr/")
from backend.helpers import get_backend_opts
from backend.daemons.job_grab import CoprJobGrab
-from backend.frontend import FrontendClient
def main():
opts = get_backend_opts()
- fc = FrontendClient(opts)
- grabber = CoprJobGrab(opts, frontend_client=fc)
+ grabber = CoprJobGrab(opts)
grabber.run()
--
2.5.0
8 years, 2 months
disable networking in Copr by default
by Pavel Raiskup
Just to be more pedantic and probably bit more secure, what about making
networking in copr opt-in (at least for new projects)?
If build of my package used internet arbitrarily, I would be warned very
early..
Also, in Koji we also do not have networking, so we can faster teach
(future) Fedora packagers.
Thanks for considering,
Pavel
8 years, 2 months
Removal of F21 chroots
by Miroslav Suchý
Fedora 21 is EOLed for one month. I plan to remove from Copr in 14 days too.
All builds will be preserved. You will not be able to submit new builds for F21.
Please update your scripts accordingly.
--
Miroslav Suchy, RHCA
Red Hat, Senior Software Engineer, #brno, #devexp, #fedora-buildsys
8 years, 2 months