[copr] master: move TODO-backend to our wiki (0c21ee8)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 0c21ee83716c320d041f470362e3c206df3b3870
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Wed Oct 2 15:44:49 2013 +0200
move TODO-backend to our wiki
>---------------------------------------------------------------
TODO-backend | 17 -----------------
1 files changed, 0 insertions(+), 17 deletions(-)
diff --git a/TODO-backend b/TODO-backend
deleted file mode 100644
index 058330e..0000000
--- a/TODO-backend
+++ /dev/null
@@ -1,17 +0,0 @@
-
-- change instance type by build request for more mem/procs/extend timeouts
- - use extra-vars?
- - need ansible 0.9?
-- auto-timer/cleanup script for old instances that may have been orphaned
-- prune out builders when we drop the number of them active
-- LOADS of fixme and catching weird conditions
-- make logging from mockremote more sane and coinsistent
-- mock configs should be pushed to instances at creation time
- - single url to repos, not mirrorlists
-- consider making each worker return job to a completed queue so the primary
- process can do other kinds of notification
-- email notifications from backend?
-- refactor mockremote/dispatcher.worker together?
-- work on a way to find and cancel a specific build that's happening other than just killing the instance
-- determine if it is properly checking the timeout from a dead instance
-- maybe dump out the PID of the worker that is running so we know which one to kill?
10 years, 7 months
[copr] master: create pid file in /var/run/copr-backend (822c814)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 822c814b4152c794acce634107a5927ef972b9e3
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 16:48:47 2013 +0200
create pid file in /var/run/copr-backend
and own it by copr user/group
and set up tmpfiles.d script so it is created again when used as tmpfs
>---------------------------------------------------------------
copr-backend.service | 2 +-
copr-be.py | 2 +-
copr.spec | 10 ++++++++++
tmpfiles.d/copr-backend.conf | 1 +
4 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/copr-backend.service b/copr-backend.service
index 0999a5c..07a8ee2 100644
--- a/copr-backend.service
+++ b/copr-backend.service
@@ -4,7 +4,7 @@ After=syslog.target network.target auditd.service
[Service]
Type=forking
-PIDFile=/var/run/copr.pid
+PIDFile=/var/run/copr-backend/copr-be.pid
ExecStart=/usr/share/copr/copr-be.py -d
ExecReload=/bin/kill -HUP $MAINPID
diff --git a/copr-be.py b/copr-be.py
index aa4f093..70749b8 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -243,7 +243,7 @@ def parse_args(args):
help="config file to use for copr-be run")
parser.add_option('-d','--daemonize', default=False, dest='daemonize',
action='store_true', help="daemonize or not")
- parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile',
+ parser.add_option('-p', '--pidfile', default='/var/run/copr-backend/copr-be.pid', dest='pidfile',
help="pid file to use for copr-be if daemonized")
parser.add_option('-x', '--exit', default=False, dest='exit_on_worker',
action='store_true', help="exit on worker failure")
diff --git a/copr.spec b/copr.spec
index a6b66f8..27f2c2e 100644
--- a/copr.spec
+++ b/copr.spec
@@ -186,6 +186,9 @@ install -d %{buildroot}%{_datadir}/copr/backend
install -d %{buildroot}%{_sysconfdir}/copr
install -d %{buildroot}%{_sysconfdir}/logrotate.d/
install -d %{buildroot}%{_unitdir}
+install -d %{buildroot}/%{_var}/log/copr-backend
+install -d %{buildroot}/%{_var}/run/copr-backend/
+install -d %{buildroot}/%{_tmpfilesdir}
cp -a backend/* %{buildroot}%{_datadir}/copr/backend
cp -a copr-be.py %{buildroot}%{_datadir}/copr/
@@ -193,12 +196,14 @@ cp -a copr-be.conf.example %{buildroot}%{_sysconfdir}/copr/copr-be.conf
cp -a backend-dist/lighttpd/* %{buildroot}%{_pkgdocdir}/lighttpd/
cp -a logrotate/* %{buildroot}%{_sysconfdir}/logrotate.d/
+cp -a tmpfiles.d/* %{buildroot}/%{_tmpfilesdir}
# for ghost files
touch %{buildroot}%{_var}/log/copr/copr.log
for i in `seq 7`; do
touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log
done
+touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid
install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
@@ -307,8 +312,12 @@ fi
%dir %attr(0755, copr, copr) %{_sharedstatedir}/copr/public_html/results
%dir %attr(0755, copr, copr) %{_var}/log/copr
%dir %attr(0755, copr, copr) %{_var}/log/copr/workers
+%dir %attr(0755, copr, copr) %{_var}/run/copr-backend
+
%ghost %{_var}/log/copr/copr.log
%ghost %{_var}/log/copr/workers/worker-*.log
+%ghost %{_var}/run/copr-backend/copr-be.pid
+
%config(noreplace) %{_sysconfdir}/logrotate.d/copr-backend
%dir %{_pkgdocdir}
%doc %{_pkgdocdir}/lighttpd
@@ -316,6 +325,7 @@ fi
%dir %{_sysconfdir}/copr
%config(noreplace) %{_sysconfdir}/copr/copr-be.conf
%{_unitdir}/copr-backend.service
+%{_tmpfilesdir}/copr-backend.conf
%{_datadir}/copr/backend
%{_datadir}/copr/copr-be.py*
diff --git a/tmpfiles.d/copr-backend.conf b/tmpfiles.d/copr-backend.conf
new file mode 100644
index 0000000..12148a7
--- /dev/null
+++ b/tmpfiles.d/copr-backend.conf
@@ -0,0 +1 @@
+D /var/run/copr-backend 0755 copr copr -
10 years, 7 months
[copr] master: add backend service file for systemd (0f5b8f2)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 0f5b8f205533db575aed689e5835e08383cc16d1
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 15:44:25 2013 +0200
add backend service file for systemd
>---------------------------------------------------------------
copr-backend.service | 12 ++++++++++++
copr.spec | 17 +++++++++++++++++
2 files changed, 29 insertions(+), 0 deletions(-)
diff --git a/copr-backend.service b/copr-backend.service
new file mode 100644
index 0000000..0999a5c
--- /dev/null
+++ b/copr-backend.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Copr Backend service
+After=syslog.target network.target auditd.service
+
+[Service]
+Type=forking
+PIDFile=/var/run/copr.pid
+ExecStart=/usr/share/copr/copr-be.py -d
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target
diff --git a/copr.spec b/copr.spec
index cab7631..a6b66f8 100644
--- a/copr.spec
+++ b/copr.spec
@@ -27,6 +27,7 @@ BuildRequires: util-linux
BuildRequires: python-setuptools
BuildRequires: python-requests
BuildRequires: python2-devel
+BuildRequires: systemd
%if 0%{?rhel} < 7 && 0%{?rhel} > 0
BuildRequires: python-argparse
%endif
@@ -58,6 +59,9 @@ Requires: python-lockfile
Requires: python-requests
Requires: logrotate
Requires: fedmsg
+Requires(post): systemd
+Requires(preun): systemd
+Requires(postun): systemd
%description backend
COPR is lightweight build system. It allows you to create new project in WebUI,
@@ -181,6 +185,7 @@ install -d %{buildroot}%{_pkgdocdir}/lighttpd/
install -d %{buildroot}%{_datadir}/copr/backend
install -d %{buildroot}%{_sysconfdir}/copr
install -d %{buildroot}%{_sysconfdir}/logrotate.d/
+install -d %{buildroot}%{_unitdir}
cp -a backend/* %{buildroot}%{_datadir}/copr/backend
cp -a copr-be.py %{buildroot}%{_datadir}/copr/
@@ -195,6 +200,8 @@ for i in `seq 7`; do
touch %{buildroot}%{_var}/log/copr/workers/worker-$i.log
done
+install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
+
#frontend
install -d %{buildroot}%{_sysconfdir}
install -d %{buildroot}%{_datadir}/copr/coprs_frontend
@@ -253,6 +260,15 @@ getent passwd copr >/dev/null || \
useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
/usr/bin/passwd -l copr >/dev/null
+%post backend
+%systemd_post copr-backend.service
+
+%preun backend
+%systemd_preun copr-backend.service
+
+%postun backend
+%systemd_postun_with_restart copr-backend.service
+
%pre frontend
getent group copr-fe >/dev/null || groupadd -r copr-fe
getent passwd copr-fe >/dev/null || \
@@ -299,6 +315,7 @@ fi
%doc %{_pkgdocdir}/playbooks
%dir %{_sysconfdir}/copr
%config(noreplace) %{_sysconfdir}/copr/copr-be.conf
+%{_unitdir}/copr-backend.service
%{_datadir}/copr/backend
%{_datadir}/copr/copr-be.py*
10 years, 7 months
[copr] master: remove daemonize option in config (94fb20c)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 94fb20cc164e60588d89feb08583a374fffe7a27
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 15:06:05 2013 +0200
remove daemonize option in config
it is not used and it will later (when it have no influence) overwrite cli option
it really does not have sense, wipe it out
>---------------------------------------------------------------
copr-be.conf.example | 4 ----
copr-be.py | 1 -
2 files changed, 0 insertions(+), 5 deletions(-)
diff --git a/copr-be.conf.example b/copr-be.conf.example
index 3d8cdf1..8068bdc 100644
--- a/copr-be.conf.example
+++ b/copr-be.conf.example
@@ -47,10 +47,6 @@ worker_logdir=/var/log/copr/workers/
# default is false
#exit_on_worker=false
-# Deamonize if true. Runs in foreground if false.
-# default is true
-#daemonize=true
-
# publish fedmsg notifications from workers if true
# default is false
#fedmsg_enabled=false
diff --git a/copr-be.py b/copr-be.py
index 0df87db..aa4f093 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -168,7 +168,6 @@ class CoprBackend(object):
opts.terminate_playbook = _get_conf(cp,'backend','terminate_playbook', '/etc/copr/terminate_playbook.yml')
opts.jobsdir = _get_conf(cp, 'backend', 'jobsdir', None)
opts.destdir = _get_conf(cp, 'backend', 'destdir', None)
- opts.daemonize = _get_conf(cp, 'backend', 'daemonize', True)
opts.exit_on_worker = _get_conf(cp, 'backend', 'exit_on_worker', False)
opts.fedmsg_enabled = _get_conf(cp, 'backend', 'fedmsg_enabled', False)
opts.sleeptime = int(_get_conf(cp, 'backend', 'sleeptime', 10))
10 years, 7 months
[copr] master: use python logging (89e58ac)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 89e58ac99a30262520e0d96c4e978add3eb36241
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 14:43:40 2013 +0200
use python logging
>---------------------------------------------------------------
copr-be.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 9e158b2..0df87db 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -10,6 +10,7 @@ import glob
import grp
import json
import lockfile
+import logging
import multiprocessing
import optparse
import os
@@ -95,7 +96,7 @@ class CoprLog(multiprocessing.Process):
os.makedirs(logdir, mode=0750)
# setup a log file to write to
- self.logfile = self.opts.logfile
+ logging.basicConfig(filename=self.opts.logfile, level=logging.DEBUG)
def log(self, event):
@@ -103,7 +104,7 @@ class CoprLog(multiprocessing.Process):
msg = '%s : %s: %s' % (when, event['who'], event['what'].strip())
try:
- open(self.logfile, 'a').write(msg + '\n')
+ logging.debug(msg)
except (IOError, OSError), e:
print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
10 years, 7 months
[copr] master: create pid file in /var/run by default (6e519b1)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6e519b1c20194252ebe11aa5e0760d067572aa9a
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 14:32:19 2013 +0200
create pid file in /var/run by default
>---------------------------------------------------------------
copr-be.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 9edf59b..9e158b2 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -243,7 +243,7 @@ def parse_args(args):
help="config file to use for copr-be run")
parser.add_option('-d','--daemonize', default=False, dest='daemonize',
action='store_true', help="daemonize or not")
- parser.add_option('-p', '--pidfile', default='copr-be.pid', dest='pidfile',
+ parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile',
help="pid file to use for copr-be if daemonized")
parser.add_option('-x', '--exit', default=False, dest='exit_on_worker',
action='store_true', help="exit on worker failure")
10 years, 7 months
[copr] master: do not create destdir (c68361f)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c68361f89795e871971945b7c585f0795ae581d0
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 14:31:54 2013 +0200
do not create destdir
it should be created in advance
>---------------------------------------------------------------
copr-be.py | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 1cb6284..9edf59b 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -94,9 +94,6 @@ class CoprLog(multiprocessing.Process):
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0750)
- if not os.path.exists(self.opts.destdir):
- os.makedirs(self.opts.destdir, mode=0755)
-
# setup a log file to write to
self.logfile = self.opts.logfile
10 years, 7 months
[copr] master: use daemon module instead of home brew function (bd21f61)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit bd21f61ec14f9315be1b3e0c670ad0776e2840d6
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 14:01:07 2013 +0200
use daemon module instead of home brew function
>---------------------------------------------------------------
copr-be.py | 89 +++++++++++++++++++++++------------------------------------
copr.spec | 2 +
2 files changed, 37 insertions(+), 54 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index fafb93d..1cb6284 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -1,18 +1,23 @@
#!/usr/bin/python -tt
-import sys
-import os
-import glob
-import time
-import multiprocessing
-from backend.dispatcher import Worker
from backend import errors
+from backend.dispatcher import Worker
from bunch import Bunch
import ConfigParser
-import optparse
+import daemon
+import glob
+import grp
import json
+import lockfile
+import multiprocessing
+import optparse
+import os
+import pwd
import requests
+import signal
+import sys
+import time
def _get_conf(cp, section, option, default):
"""to make returning items from config parser less irritating"""
@@ -147,6 +152,7 @@ class CoprBackend(object):
self.workers = []
self.added_jobs = []
+ self.abort = False
def event(self, what):
self.events.put({'when':time.time(), 'who':'main', 'what':what})
@@ -192,8 +198,8 @@ class CoprBackend(object):
def run(self):
- abort = False
- while not abort:
+ self.abort = False
+ while not self.abort:
# re-read config into opts
self.opts = self.read_conf()
@@ -227,46 +233,12 @@ class CoprBackend(object):
time.sleep(self.opts.sleeptime)
-# lifted from certmaster
-def daemonize(pidfile=None):
- """
- Daemonize this process with the UNIX double-fork trick.
- Writes the new PID to the provided file name if not None.
- """
-
- cur_umask = os.umask(077)
- os.umask(cur_umask)
-
- pid = os.fork()
- if pid > 0:
- sys.exit(0)
- os.chdir("/")
- os.setsid()
- os.umask(cur_umask)
- pid = os.fork()
-
- os.close(0)
- os.close(1)
- os.close(2)
-
- # The standard I/O file descriptors are redirected to /dev/null by default.
- if (hasattr(os, "devnull")):
- REDIRECT_TO = os.devnull
- else:
- REDIRECT_TO = "/dev/null"
-
- # based on http://code.activestate.com/recipes/278731/
- os.open(REDIRECT_TO, os.O_RDWR) # standard input (0)
-
- os.dup2(0, 1) # standard output (1)
- os.dup2(0, 2) # standard error (2)
-
-
-
- if pid > 0:
- if pidfile is not None:
- open(pidfile, "w").write(str(pid))
- sys.exit(0)
+ def terminate(self):
+ """ Cleanup backend processes (just workers for now) """
+ self.abort = True
+ for w in self.workers:
+ self.workers.remove(w)
+ w.terminate()
def parse_args(args):
parser = optparse.OptionParser('\ncopr-be [options]')
@@ -298,14 +270,23 @@ def main(args):
try:
cbe = CoprBackend(opts.config_file, ext_opts=opts)
- if opts.daemonize:
- daemonize(opts.pidfile)
- cbe.run()
+ context = daemon.DaemonContext(
+ pidfile=lockfile.FileLock(opts.pidfile),
+ gid = grp.getgrnam('copr').gr_gid,
+ uid = pwd.getpwnam('copr').pw_uid,
+ detach_process = opts.daemonize,
+ umask = 077,
+ signal_map = {
+ signal.SIGTERM: 'terminate',
+ signal.SIGHUP: 'terminate',
+ },
+ )
+ with context:
+ cbe.run()
except Exception, e:
print 'Killing/Dying'
if 'cbe' in locals():
- for w in cbe.workers:
- w.terminate()
+ cbe.terminate()
raise
except KeyboardInterrupt, e:
pass
diff --git a/copr.spec b/copr.spec
index a636425..cab7631 100644
--- a/copr.spec
+++ b/copr.spec
@@ -53,6 +53,8 @@ Requires: mock
Requires: yum-utils
Requires: createrepo
Requires: python-bunch
+Requires: python-daemon
+Requires: python-lockfile
Requires: python-requests
Requires: logrotate
Requires: fedmsg
10 years, 7 months
[copr] master: fix default location of copr-be.conf (87cc55a)
by Miroslav Suchý
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 87cc55a1cbaa3cd9669195a3300e13964f640d5e
Author: Miroslav Suchý <msuchy(a)redhat.com>
Date: Tue Oct 1 10:04:17 2013 +0200
fix default location of copr-be.conf
>---------------------------------------------------------------
copr-be.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index c836592..fafb93d 100755
--- a/copr-be.py
+++ b/copr-be.py
@@ -270,7 +270,7 @@ def daemonize(pidfile=None):
def parse_args(args):
parser = optparse.OptionParser('\ncopr-be [options]')
- parser.add_option('-c', '--config', default='/etc/copr-be.conf', dest='config_file',
+ parser.add_option('-c', '--config', default='/etc/copr/copr-be.conf', dest='config_file',
help="config file to use for copr-be run")
parser.add_option('-d','--daemonize', default=False, dest='daemonize',
action='store_true', help="daemonize or not")
10 years, 7 months