[copr] skvidal-backend: move the config reread out of the queuesize change - so we know it will happen in the loop even when no jobs are happening (00ccbef)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 00ccbefe32ed5a4f5827e4d7831a85f4752d597e
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:18:49 2012 -0500
move the config reread out of the queuesize change - so we know it will happen in the loop
even when no jobs are happening
>---------------------------------------------------------------
copr-be.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index e5128d9..6564134 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -134,10 +134,11 @@ class CoprBackend(object):
self.added_jobs.append(n)
self.log('adding to work queue id %s' % n)
+ # re-read config into opts
+ self.opts = self.read_conf()
+
if self.jobs.qsize():
self.log("# jobs in queue: %s" % self.jobs.qsize())
- # re-read config into opts
- self.opts = self.read_conf()
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
self.log("Spinning up more workers for jobs")
11 years, 4 months
[copr] skvidal-backend: when we startup make sure our destdir exists when a worker dies unexpectedly - make sure we kill it and get rid of it (9da44f0)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 9da44f0b3c2417e53c9eadf9f44d17d3e5f272bf
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:16:25 2012 -0500
when we startup make sure our destdir exists
when a worker dies unexpectedly - make sure we kill it and get rid of it
>---------------------------------------------------------------
copr-be.py | 11 +++++++----
1 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 27ceb03..e5128d9 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -37,6 +37,9 @@ class CoprBackend(object):
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0750)
+ if not os.path.exists(self.opts.destdir):
+ os.makedirs(self.opts.destdir, mode=0755)
+
# setup a log file to write to
self.logfile = self.opts.logfile
self.log("Starting up new copr-be instance")
@@ -106,8 +109,6 @@ class CoprBackend(object):
except requests.RequestException, e:
self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
else:
-
-
r_json = json.loads(r.content) # using old requests on el6 :(
if 'builds' in r_json:
self.log('%s jobs returned' % len(r_json['builds']))
@@ -131,7 +132,7 @@ class CoprBackend(object):
if n not in self.added_jobs:
self.jobs.put(f)
self.added_jobs.append(n)
- self.log('adding %s' % n)
+ self.log('adding to work queue id %s' % n)
if self.jobs.qsize():
self.log("# jobs in queue: %s" % self.jobs.qsize())
@@ -159,7 +160,9 @@ class CoprBackend(object):
self.log('Worker %d died unexpectedly' % w.worker_num)
if self.opts.exit_on_worker:
raise errors.CoprBackendError, "Worker died unexpectedly, exiting"
-
+ else:
+ self.workers.remove(w) # it is not working anymore
+ w.terminate() # kill it with a fire
time.sleep(self.opts.sleeptime)
11 years, 4 months
[copr] skvidal-backend: older python-requests doesn't have attribute :( (02f9e18)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 02f9e18594e1a03d9bc127da601e45aebaca8b0c
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:00:11 2012 -0500
older python-requests doesn't have attribute :(
>---------------------------------------------------------------
copr-be.py | 9 ++++++---
1 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 8ab8470..27ceb03 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -106,10 +106,13 @@ class CoprBackend(object):
except requests.RequestException, e:
self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
else:
- if 'builds' in r.json:
- self.log('%s jobs returned' % len(r.json['builds']))
+
+
+ r_json = json.loads(r.content) # using old requests on el6 :(
+ if 'builds' in r_json:
+ self.log('%s jobs returned' % len(r_json['builds']))
count = 0
- for b in r.json['builds']:
+ for b in r_json['builds']:
if 'id' in b:
jobfile = self.opts.jobsdir + '/%s.json' % b['id']
if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
11 years, 4 months
[copr] skvidal-backend: - saving the jobs out per-build - so don't expect a 'builds' dictionary - fetch the jobs from the front end (2eb8d67)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 2eb8d67a3a2ebbc509e8c25985d44b54287d8f62
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 01:52:44 2012 -0500
- saving the jobs out per-build - so don't expect a 'builds' dictionary
- fetch the jobs from the front end
>---------------------------------------------------------------
backend/dispatcher.py | 3 +--
copr-be.py | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index b9c1251..fc9a958 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -153,8 +153,7 @@ class Worker(multiprocessing.Process):
def parse_job(self, jobfile):
# read the json of the job in
# break out what we need return a bunch of the info we need
- d = json.load(open(jobfile))
- build = d['builds'][0]
+ build = json.load(open(jobfile))
jobdata = Bunch()
jobdata.pkgs = build['pkgs'].split(' ')
jobdata.repos = [r for r in build['repos'].split(' ') if r.strip() ]
diff --git a/copr-be.py b/copr-be.py
index bd26444..8ab8470 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -11,6 +11,8 @@ from backend import errors
from bunch import Bunch
import ConfigParser
import optparse
+import json
+import requests
def _get_conf(cp, section, option, default):
"""to make returning items from config parser less irritating"""
@@ -97,10 +99,30 @@ class CoprBackend(object):
print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
+ def fetch_jobs(self):
+ self.log('fetching jobs')
+ try:
+ r = requests.get('%s/waiting_builds/' % self.opts.frontend_url) # auth stuff here? maybe/maybenot
+ except requests.RequestException, e:
+ self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
+ else:
+ if 'builds' in r.json:
+ self.log('%s jobs returned' % len(r.json['builds']))
+ count = 0
+ for b in r.json['builds']:
+ if 'id' in b:
+ jobfile = self.opts.jobsdir + '/%s.json' % b['id']
+ if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
+ count += 1
+ open(jobfile, 'w').write(json.dumps(b))
+ self.log('Wrote job: %s' % b['id'])
+ self.log('New jobs: %s' % count)
+
def run(self):
abort = False
while not abort:
+ self.fetch_jobs()
for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
n = os.path.basename(f).replace('.json', '')
if n not in self.added_jobs:
11 years, 4 months
[copr] skvidal-backend: make sure we're using whatever current umask is and we don't force into lesser levels (6d065fa)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 6d065fa3965f0974cdc57b6d9a54f4a7205f56bf
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 17:06:37 2012 -0500
make sure we're using whatever current umask is and we don't force into lesser levels
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
copr-be.py | 6 +++++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 16e0825..b9c1251 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -216,7 +216,7 @@ class Worker(multiprocessing.Process):
# setup our target dir locally
if not os.path.exists(chroot_destdir):
try:
- os.makedirs(chroot_destdir, mode=0755)
+ os.makedirs(chroot_destdir)
except (OSError, IOError), e:
msg = "Could not make results dir for job: %s - %s" % (chroot_destdir, str(e))
self.callback.log(msg)
diff --git a/copr-be.py b/copr-be.py
index 9cd2bba..bd26444 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -145,12 +145,15 @@ def daemonize(pidfile=None):
Writes the new PID to the provided file name if not None.
"""
+ cur_umask = os.umask(077)
+ os.umask(cur_umask)
+
pid = os.fork()
if pid > 0:
sys.exit(0)
os.chdir("/")
os.setsid()
- os.umask(077)
+ os.umask(cur_umask)
pid = os.fork()
os.close(0)
@@ -223,6 +226,7 @@ if __name__ == '__main__':
main(sys.argv[1:])
except Exception, e:
print "ERROR: %s - %s" % (str(type(e)), str(e))
+ # FIXME - maybe check on daemonize and do this as a syslog.syslog() call?
sys.exit(1)
except KeyboardInterrupt, e:
print "\nUser cancelled, may need cleanup\n"
11 years, 4 months
[copr] skvidal-backend: when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/ (9f65c8f)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 9f65c8fd7a6cd0ce5931d61dce51b71ad5b3af0f
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 16:29:00 2012 -0500
when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/
>---------------------------------------------------------------
copr-be.py | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 9a5e3a4..9cd2bba 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -191,6 +191,7 @@ def parse_args(args):
if not os.path.exists(opts.config_file):
print "No config file found at: %s" % opts.config_file
sys.exit(1)
+ opts.config_file = os.path.abspath(opts.config_file)
ret_opts = Bunch()
for o in ('daemonize', 'exit_on_worker', 'pidfile', 'config_file'):
@@ -204,9 +205,9 @@ def main(args):
opts = parse_args(args)
try:
+ cbe = CoprBackend(opts.config_file, ext_opts=opts)
if opts.daemonize:
daemonize(opts.pidfile)
- cbe = CoprBackend(opts.config_file, ext_opts=opts)
cbe.run()
except Exception, e:
print 'Killing/Dying'
11 years, 4 months
[copr] skvidal-backend: trying to figure out why the daemonized process dies (7ae63a9)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 7ae63a9178245688bb23ac7168fbe5546918187b
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 16:17:31 2012 -0500
trying to figure out why the daemonized process dies
>---------------------------------------------------------------
copr-be.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 5b35905..9a5e3a4 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -204,9 +204,9 @@ def main(args):
opts = parse_args(args)
try:
- cbe = CoprBackend(opts.config_file, ext_opts=opts)
if opts.daemonize:
daemonize(opts.pidfile)
+ cbe = CoprBackend(opts.config_file, ext_opts=opts)
cbe.run()
except Exception, e:
print 'Killing/Dying'
11 years, 4 months
[copr] skvidal-backend: - make pid file default to cwd unless specified - better log output for the building - fixme notes (2a047db)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 2a047db2d68b9439a6d63d01f1f31845f5a6a7a7
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 16:04:22 2012 -0500
- make pid file default to cwd unless specified
- better log output for the building
- fixme notes
>---------------------------------------------------------------
backend/dispatcher.py | 7 ++++++-
copr-be.py | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index f96bdb1..16e0825 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -191,6 +191,10 @@ class Worker(multiprocessing.Process):
# parse the job json into our info
job = self.parse_job(jobfile)
+ # FIXME
+ # this is our best place to sanity check the job before starting
+ # up any longer process
+
job.jobfile = jobfile
# spin up our build instance
@@ -227,7 +231,7 @@ class Worker(multiprocessing.Process):
# start the build - most importantly license checks.
- self.callback.log('Calling mockremote with builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
+ self.callback.log('Starting build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
try:
chrootlogfile = chroot_destdir + '/mockremote.log'
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
@@ -239,6 +243,7 @@ class Worker(multiprocessing.Process):
# record and break
self.callback.log('%s - %s' % (ip, e))
status = 0 # failure
+ self.callback.log('Finished build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
job.ended_on = time.time()
job.status = status
diff --git a/copr-be.py b/copr-be.py
index d98feea..5b35905 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -182,7 +182,7 @@ def parse_args(args):
help="config file to use for copr-be run")
parser.add_option('-d','--daemonize', default=False, dest='daemonize',
action='store_true', help="daemonize or not")
- parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile',
+ parser.add_option('-p', '--pidfile', default='copr-be.pid', dest='pidfile',
help="pid file to use for copr-be if daemonized")
parser.add_option('-x', '--exit', default=False, dest='exit_on_worker',
action='store_true', help="exit on worker failure")
11 years, 4 months
[copr] skvidal-backend: spaces in rsync command for name of copr :-/ (858a533)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 858a5331b1902764f7ee9a3838c7c1a2c8cfe10a
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 15:47:03 2012 -0500
spaces in rsync command for name of copr :-/
>---------------------------------------------------------------
backend/mockremote.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/backend/mockremote.py b/backend/mockremote.py
index 7698de9..3d39f3d 100755
--- a/backend/mockremote.py
+++ b/backend/mockremote.py
@@ -329,6 +329,7 @@ class Builder(object):
success = False
rpd = self._get_remote_pkg_dir(pkg)
+ destdir = "'" + destdir.replace("'", "'\\''") + "'" # make spaces work w/our rsync command below :(
# build rsync command line from the above
remote_src = '%s@%s:%s' % (self.username, self.hostname, rpd)
ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
11 years, 4 months
[copr] skvidal-backend: still need something like a useful output when it does break (58c0441)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 58c0441ccecf4694cc45a82f43fcf372a9090528
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 15:14:14 2012 -0500
still need something like a useful output when it does break
>---------------------------------------------------------------
copr-be.py | 11 ++++++-----
1 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 3be1828..d98feea 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -28,7 +28,7 @@ class CoprBackend(object):
raise errors.CoprBackendError, "Must specify config_file"
self.config_file = config_file
- self._ext_opts = ext_opts # to stow our cli options for read_conf()
+ self.ext_opts = ext_opts # to stow our cli options for read_conf()
self.opts = self.read_conf()
logdir = os.path.dirname(self.opts.logfile)
@@ -79,8 +79,8 @@ class CoprBackend(object):
if not opts.jobsdir or not opts.destdir:
raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration"
- if self._ext_opts:
- for v in self._ext_opts:
+ if self.ext_opts:
+ for v in self.ext_opts:
setattr(opts, v, self.ext_opts.get(v))
return opts
@@ -213,8 +213,9 @@ def main(args):
if 'cbe' in locals():
for w in cbe.workers:
w.terminate()
-
-
+ raise
+ except KeyboardInterrupt, e:
+ pass
if __name__ == '__main__':
try:
11 years, 4 months