Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 92131b63745ce85c491691a2963f71317931d60b
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu May 28 15:00:32 2015 +0200
[backend] config cleanup
---------------------------------------------------------------
backend/backend/daemons/backend.py | 2 -
backend/backend/daemons/dispatcher.py | 11 +--
backend/backend/daemons/job_grab.py | 5 +-
backend/backend/daemons/log.py | 77 -------------------------
backend/backend/frontend.py | 4 +-
backend/backend/helpers.py | 20 +------
backend/conf/copr-be.conf.example | 23 -------
backend/conf/copr-be.local.conf | 19 ------
backend/tests/deamons/test_backend.py | 9 +---
backend/tests/deamons/test_dispatcher.py | 10 +--
backend/tests/deamons/test_job_grab.py | 4 +-
backend/tests/deamons/test_log.py | 6 +-
backend/tests/test_frontend.py | 2 +-
backend/tests/vm_manager/test_check.py | 1 -
backend/tests/vm_manager/test_event_handle.py | 2 -
backend/tests/vm_manager/test_manager.py | 1 -
backend/tests/vm_manager/test_spawn.py | 1 -
backend/tests/vm_manager/test_terminate.py | 2 -
18 files changed, 19 insertions(+), 180 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 21f9958..bcf4d44 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -69,8 +69,6 @@ class CoprBackend(object):
# what:str}
self.frontend_client = FrontendClient(self.opts)
self.is_running = False
- if not os.path.exists(self.opts.worker_logdir):
- os.makedirs(self.opts.worker_logdir, mode=0o750)
self.log = get_redis_logger(self.opts, "backend.main",
"backend")
diff --git a/backend/backend/daemons/dispatcher.py
b/backend/backend/daemons/dispatcher.py
index 1b43ede..7cb5c65 100644
--- a/backend/backend/daemons/dispatcher.py
+++ b/backend/backend/daemons/dispatcher.py
@@ -47,29 +47,26 @@ class Worker(multiprocessing.Process):
multiprocessing.Process.__init__(self, name="worker-builder")
self.opts = opts
+ self.worker_num = worker_num
+ self.group_id = group_id
+
+ self.log = get_redis_logger(self.opts, self.logger_name, "worker")
# job management stuff
self.task_queue = Queue("copr-be-{0}".format(str(group_id)))
self.task_queue.connect()
# event queue for communicating back to dispatcher
- self.worker_num = worker_num
- self.group_id = group_id
-
self.kill_received = False
self.lock = lock
self.frontend_client = frontend_client
-
- self.log = get_redis_logger(self.opts, self.logger_name, "worker")
-
self.vm_name = None
self.vm_ip = None
self.rc = None
self.vmm = VmManager(self.opts)
-
@property
def logger_name(self):
return "backend.worker-{}-{}".format(self.group_name, self.worker_num)
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index 246e3e0..36aea77 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -147,10 +147,11 @@ class CoprJobGrab(Process):
Retrieve tasks from frontend and runs appropriate handlers
"""
try:
- r = get("{0}/waiting/".format(self.opts.frontend_url),
+ r =
get("{0}/backend/waiting/".format(self.opts.frontend_base_url),
auth=("user", self.opts.frontend_auth))
except RequestException as e:
- self.log.exception("Error retrieving jobs from {}:
{}".format(self.opts.frontend_url, e))
+ self.log.exception("Error retrieving jobs from {}: {}"
+ .format(self.opts.frontend_base_url, e))
return
try:
diff --git a/backend/backend/daemons/log.py b/backend/backend/daemons/log.py
index 8a5dedb..f74962e 100644
--- a/backend/backend/daemons/log.py
+++ b/backend/backend/daemons/log.py
@@ -21,83 +21,6 @@ from .. import helpers
from ..constants import default_log_format
-class CoprBackendLog(Process):
-
- """Log mechanism where items from the events queue get recorded
-
- :param Bunch opts: backend config
- :param events: multiprocessing.Queue to listen
- for events from other backend components
-
- """
-
- def __init__(self, opts, events):
-
- # base class initialization
- Process.__init__(self, name="logger")
-
- self.opts = opts
- self.events = events
-
- logdir = os.path.dirname(self.opts.logfile)
- if not os.path.exists(logdir):
- os.makedirs(logdir, mode=0o750)
-
- def setup_log_handler(self):
- """
- Configures standard python logger
- """
- sys.stderr.write("Running setup handler {} \n".format(self.opts))
- # setup a log file to write to
- logging.basicConfig(filename=self.opts.logfile + "_old.log",
level=logging.DEBUG)
-
- self.log({"when": time.time(), "who":
self.__class__.__name__,
- "what": "Logger initiated"})
-
- def log(self, event):
- """
- Format event into the log message
-
- :param event: dict-like object
-
- Expected **event** keys:
- - `when`: unixtime
- - `who`: event producer [worker|logger|job|main]
- - `what`: content
- """
- when = time.strftime("%F %T", time.gmtime(event["when"]))
- msg = "{0} : {1}: {2}".format(when,
- event["who"],
- event["what"].strip())
- try:
- if self.opts.verbose:
- sys.stderr.write("{0}\n".format(msg))
- sys.stderr.flush()
- # logging.debug(msg)
-
- except (IOError, OSError) as e:
-
- sys.stderr.write("Could not write to logfile {0} - {1}\n".format(
- self.opts.logfile, e))
-
- # event format is a dict {when:time, who:[worker|logger|job|main],
- # what:str}
- def run(self):
- """
- Starts logger process
- """
- setproctitle("CoprLog")
- self.setup_log_handler()
-
- try:
- while True:
- event = self.events.get()
- if "when" in event and "who" in event and
"what" in event:
- self.log(event)
- except KeyboardInterrupt:
- return
-
-
level_map = {
"info": logging.INFO,
"debug": logging.DEBUG,
diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py
index 86d5e22..52c91b6 100644
--- a/backend/backend/frontend.py
+++ b/backend/backend/frontend.py
@@ -10,7 +10,7 @@ class FrontendClient(object):
def __init__(self, opts):
super(FrontendClient, self).__init__()
- self.frontend_url = opts.frontend_url
+ self.frontend_url = "{}/backend".format(opts.frontend_base_url)
self.frontend_auth = opts.frontend_auth
self.msg = None
@@ -21,7 +21,7 @@ class FrontendClient(object):
"""
headers = {"content-type": "application/json"}
- url = "{0}/{1}/".format(self.frontend_url, url_path)
+ url = "{}/{}/".format(self.frontend_url, url_path)
auth = ("user", self.frontend_auth)
self.msg = None
diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py
index 08b7794..3755113 100644
--- a/backend/backend/helpers.py
+++ b/backend/backend/helpers.py
@@ -104,15 +104,10 @@ class BackendConfigReader(object):
opts = Bunch()
opts.results_baseurl = _get_conf(
- cp, "backend", "results_baseurl",
"http://copr")
+ cp, "backend", "results_baseurl",
"http://copr-be")
- # TODO: this should be built from frontend_base_url + '/backend'
- opts.frontend_url = _get_conf(
- cp, "backend", "frontend_url",
"http://coprs/rest/api")
-
- # We need this to access public api
opts.frontend_base_url = _get_conf(
- cp, "backend", "frontend_base_url",
"http://coprs/")
+ cp, "backend", "frontend_base_url",
"http://copr-fe")
opts.frontend_auth = _get_conf(
cp, "backend", "frontend_auth",
"PASSWORDHERE")
@@ -205,21 +200,10 @@ class BackendConfigReader(object):
cp, "backend", "log_dir", "/var/log/copr/")
opts.log_level = _get_conf(
cp, "backend", "log_level", "info")
- opts.logfile = _get_conf(
- cp, "backend", "logfile",
"/var/log/copr/backend.log")
- opts.error_logfile = _get_conf(
- cp, "backend", "error_logfile",
"/var/log/copr/backend_error.log")
opts.verbose = _get_conf(
cp, "backend", "verbose", False, mode="bool")
- opts.worker_logdir = _get_conf(
- cp, "backend", "worker_logdir",
"/var/log/copr/workers/")
- opts.terminate_vars = _get_conf(cp, "backend",
"terminate_vars", "").split(",")
opts.prune_days = _get_conf(cp, "backend", "prune_days",
None, mode="int")
- opts.prune_script = _get_conf(cp, "backend", "prune_script",
None, mode="path")
-
- opts.spawn_in_advance = _get_conf(
- cp, "backend", "spawn_in_advance", False,
mode="bool")
# ssh options
opts.ssh = Bunch()
diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example
index 4a34206..1f9f768 100644
--- a/backend/conf/copr-be.conf.example
+++ b/backend/conf/copr-be.conf.example
@@ -5,7 +5,6 @@
results_baseurl=https://copr-be.cloud.fedoraproject.org/results
# default is
http://coprs/rest/api
-frontend_url=https://copr-fe.cloud.fedoraproject.org/backend
frontend_base_url=http://copr-fe-dev.cloud.fedoraproject.org
# must have same value as BACKEND_PASSWORD from have frontend in /etc/copr/copr.conf
@@ -43,16 +42,6 @@ group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml
group0_max_workers=8
group0_max_vm_total=2
-# That variables will be send to
-# terminate_playbook ansible script.
-# Currently supported parameters:
-# - ip
-# IP address of builder which should be terminated
-# - vm_name
-# name of VM (if provided by spawn script)
-# default is None (empty)
-#terminate_vars=ip
-
# directory where results are stored
# should be accessible from web using 'results_baseurl' URL
# no default
@@ -63,13 +52,6 @@ destdir=/var/lib/copr/public_html/results
# default is 10
sleeptime=30
-# path to log file
-# default is /var/log/copr/backend.log
-logfile=/var/log/copr/backend.log
-
-# default is /var/log/copr/workers/
-worker_logdir=/var/log/copr/workers/
-
# exit on worker failure
# default is false
#exit_on_worker=false
@@ -90,11 +72,6 @@ worker_logdir=/var/log/copr/workers/
# minimum age for builds to be pruned
prune_days=14
-# Spawn builder in advance, before we get task?
-# It save time, but consume resources even when
-# nothing is in queue
-#spawn_in_advance=false
-
[builder]
# default is 1800
timeout=3600
diff --git a/backend/conf/copr-be.local.conf b/backend/conf/copr-be.local.conf
index 0eeb2bf..7897e1c 100644
--- a/backend/conf/copr-be.local.conf
+++ b/backend/conf/copr-be.local.conf
@@ -5,7 +5,6 @@
results_baseurl=https://copr-be.cloud.fedoraproject.org/results
# default is
http://coprs/rest/api
-frontend_url=https://copr-fe.cloud.fedoraproject.org/backend
frontend_base_url=http://copr-fe-dev.cloud.fedoraproject.org
# must have same value as BACKEND_PASSWORD from have frontend in /etc/copr/copr.conf
@@ -41,17 +40,6 @@ group0_spawn_playbook=/srv/copr-work/provision/builderpb-PC.yml
group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml
group0_max_workers=8
-
-# That variables will be send to
-# terminate_playbook ansible script.
-# Currently supported parameters:
-# - ip
-# IP address of builder which should be terminated
-# - vm_name
-# name of VM (if provided by spawn script)
-# default is None (empty)
-#terminate_vars=ip
-
# directory where results are stored
# should be accessible from web using 'results_baseurl' URL
# no default
@@ -65,13 +53,6 @@ sleeptime=30
# default is 8
num_workers=5
-# path to log file
-# default is /var/log/copr/backend.log
-logfile=/tmp/copr/backend.log
-
-# default is /var/log/copr/workers/
-worker_logdir=/tmp/log/copr/workers/
-
# exit on worker failure
# default is false
#exit_on_worker=false
diff --git a/backend/tests/deamons/test_backend.py
b/backend/tests/deamons/test_backend.py
index e723efe..d36797a 100644
--- a/backend/tests/deamons/test_backend.py
+++ b/backend/tests/deamons/test_backend.py
@@ -72,7 +72,6 @@ class TestBackend(object):
self.mp_patcher =
mock.patch("backend.daemons.backend.multiprocessing")
self.mc_mp = self.mp_patcher.start()
- self.worker_logdir = os.path.join(self.tmp_dir_path, "workers")
self.config_file = "/dev/null/copr.conf"
self.ext_opts = {}
@@ -80,7 +79,6 @@ class TestBackend(object):
self.bc_obj = MagicMock()
self.opts = Bunch(
- worker_logdir=self.worker_logdir,
build_groups=[
{
"id": 0,
@@ -97,7 +95,6 @@ class TestBackend(object):
],
exit_on_worker=False,
sleeptime=1,
-
frontend_url="http://example.com/backend",
frontend_base_url="http://example.com",
frontend_auth="foobar",
@@ -154,12 +151,7 @@ class TestBackend(object):
self.be = CoprBackend(None, self.ext_opts)
def test_constructor(self):
-
- assert not os.path.exists(self.worker_logdir)
self.init_be()
- assert os.path.exists(self.worker_logdir)
- # import ipdb; ipdb.set_trace()
-
assert self.be.config_reader == self.bc_obj
assert self.bc_obj.read.called
@@ -173,6 +165,7 @@ class TestBackend(object):
def test_clean_task_queue_ok(self, init_be):
mc_queue = MagicMock(length=5)
+
def decr():
mc_queue.length -= 1
diff --git a/backend/tests/deamons/test_dispatcher.py
b/backend/tests/deamons/test_dispatcher.py
index 143043e..2f5221e 100644
--- a/backend/tests/deamons/test_dispatcher.py
+++ b/backend/tests/deamons/test_dispatcher.py
@@ -115,22 +115,19 @@ class TestDispatcher(object):
self.terminate_pb = "/terminate.yml"
self.opts = Bunch(
ssh=Bunch(transport="paramiko"),
- spawn_in_advance=False,
-
frontend_url="http://example.com/",
+ frontend_base_url="http://example.com",
frontend_auth="12345678",
build_groups={
- "3": {
+ 3: {
"spawn_playbook": self.spawn_pb,
"terminate_playbook": self.terminate_pb,
"name": "3"
}
},
- terminate_vars=[],
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
- worker_logdir=self.tmp_dir_path,
timeout=1800,
destdir=self.tmp_dir_path,
results_baseurl="/tmp",
@@ -142,7 +139,7 @@ class TestDispatcher(object):
self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)
self.worker_num = 2
- self.group_id = "3"
+ self.group_id = 3
self.ip = "192.168.1.1"
self.worker_callback = MagicMock()
@@ -169,7 +166,6 @@ class TestDispatcher(object):
self.worker.vmm = MagicMock()
-
def set_ip(*args, **kwargs):
self.worker.vm_ip = self.vm_ip
diff --git a/backend/tests/deamons/test_job_grab.py
b/backend/tests/deamons/test_job_grab.py
index a5736fe..fc06315 100644
--- a/backend/tests/deamons/test_job_grab.py
+++ b/backend/tests/deamons/test_job_grab.py
@@ -85,7 +85,6 @@ class TestJobGrab(object):
self.log_file = os.path.join(self.log_dir, "copr.log")
self.opts = Bunch(
- logfile=self.log_file,
verbose=False,
build_groups=[
{"id": 0, "name": "x86",
@@ -95,8 +94,7 @@ class TestJobGrab(object):
"max_vm_per_user": 5},
],
destdir="/dev/null",
-
frontend_base_url="http://example.com/",
-
frontend_url="http://example.com/backend",
+ frontend_base_url="http://example.com",
frontend_auth="foobar",
results_baseurl="http://example.com/results/",
sleeptime=1,
diff --git a/backend/tests/deamons/test_log.py b/backend/tests/deamons/test_log.py
index 30002c2..8b4ed52 100644
--- a/backend/tests/deamons/test_log.py
+++ b/backend/tests/deamons/test_log.py
@@ -20,7 +20,7 @@ else:
import pytest
import backend.daemons.log as log_module
-from backend.daemons.log import CoprBackendLog, RedisLogHandler
+from backend.daemons.log import RedisLogHandler
@pytest.yield_fixture
@@ -52,9 +52,7 @@ class TestLog(object):
self.log_dir = os.path.join(self.tmp_dir_path, "copr")
self.log_file = os.path.join(self.log_dir, "copr.log")
self.opts = Bunch(
- logfile=self.log_file,
verbose=False,
-
log_dir=self.log_dir
)
print("\n log dir: {}".format(self.log_dir))
@@ -72,7 +70,7 @@ class TestLog(object):
@pytest.fixture
def init_log(self):
- self.cbl = CoprBackendLog(self.opts, self.queue)
+ pass
# todo: replace with RedisLogHandler + helpers.get_redis_logger
# def test_constructor(self):
diff --git a/backend/tests/test_frontend.py b/backend/tests/test_frontend.py
index ad85e4b..b2ee80f 100644
--- a/backend/tests/test_frontend.py
+++ b/backend/tests/test_frontend.py
@@ -35,7 +35,7 @@ class TestFrontendClient(object):
def setup_method(self, method):
self.opts = Bunch(
-
frontend_url="http://example.com/",
+
frontend_base_url="http://example.com/",
frontend_auth="12345678",
)
self.fc = FrontendClient(self.opts)
diff --git a/backend/tests/vm_manager/test_check.py
b/backend/tests/vm_manager/test_check.py
index e7d6172..2f20080 100644
--- a/backend/tests/vm_manager/test_check.py
+++ b/backend/tests/vm_manager/test_check.py
@@ -84,7 +84,6 @@ class TestChecker(object):
"terminate_playbook": self.terminate_pb_path,
"name": "base",
"archs": ["i386", "x86_64"],
- # "terminate_vars": ["vm_name", "ip"],
}
},
diff --git a/backend/tests/vm_manager/test_event_handle.py
b/backend/tests/vm_manager/test_event_handle.py
index 9db7f12..f1e7192 100644
--- a/backend/tests/vm_manager/test_event_handle.py
+++ b/backend/tests/vm_manager/test_event_handle.py
@@ -89,14 +89,12 @@ class TestEventHandle(object):
"name": "base",
"archs": ["i386", "x86_64"],
"vm_max_check_fails": 2,
- # "terminate_vars": ["vm_name", "ip"],
}
},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
- # worker_logdir=self.,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
diff --git a/backend/tests/vm_manager/test_manager.py
b/backend/tests/vm_manager/test_manager.py
index 270d087..787cb6a 100644
--- a/backend/tests/vm_manager/test_manager.py
+++ b/backend/tests/vm_manager/test_manager.py
@@ -65,7 +65,6 @@ class TestManager(object):
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
- # worker_logdir=self.,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
diff --git a/backend/tests/vm_manager/test_spawn.py
b/backend/tests/vm_manager/test_spawn.py
index 729315b..3b9e849 100644
--- a/backend/tests/vm_manager/test_spawn.py
+++ b/backend/tests/vm_manager/test_spawn.py
@@ -87,7 +87,6 @@ class TestSpawner(object):
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
- # worker_logdir=self.,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",
diff --git a/backend/tests/vm_manager/test_terminate.py
b/backend/tests/vm_manager/test_terminate.py
index 6ef716c..ef79ad4 100644
--- a/backend/tests/vm_manager/test_terminate.py
+++ b/backend/tests/vm_manager/test_terminate.py
@@ -82,14 +82,12 @@ class TestTerminate(object):
"terminate_playbook": self.terminate_pb_path,
"name": "base",
"archs": ["i386", "x86_64"],
- # "terminate_vars": ["vm_name", "ip"],
}
},
fedmsg_enabled=False,
sleeptime=0.1,
do_sign=True,
- # worker_logdir=self.,
timeout=1800,
# destdir=self.tmp_dir_path,
results_baseurl="/tmp",