December 2012 - copr-devel - Fedora mailing-lists

[copr] skvidal-backend: when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/ (9f65c8f)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 9f65c8fd7a6cd0ce5931d61dce51b71ad5b3af0f Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:29:00 2012 -0500 when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/ >--------------------------------------------------------------- copr-be.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 9a5e3a4..9cd2bba 100644 --- a/copr-be.py +++ b/copr-be.py @@ -191,6 +191,7 @@ def parse_args(args): if not os.path.exists(opts.config_file): print "No config file found at: %s" % opts.config_file sys.exit(1) + opts.config_file = os.path.abspath(opts.config_file) ret_opts = Bunch() for o in ('daemonize', 'exit_on_worker', 'pidfile', 'config_file'): @@ -204,9 +205,9 @@ def main(args): opts = parse_args(args) try: + cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) - cbe = CoprBackend(opts.config_file, ext_opts=opts) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: trying to figure out why the daemonized process dies (7ae63a9)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 7ae63a9178245688bb23ac7168fbe5546918187b Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:17:31 2012 -0500 trying to figure out why the daemonized process dies >--------------------------------------------------------------- copr-be.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 5b35905..9a5e3a4 100644 --- a/copr-be.py +++ b/copr-be.py @@ -204,9 +204,9 @@ def main(args): opts = parse_args(args) try: - cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) + cbe = CoprBackend(opts.config_file, ext_opts=opts) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: - make pid file default to cwd unless specified - better log output for the building - fixme notes (2a047db)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 2a047db2d68b9439a6d63d01f1f31845f5a6a7a7 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:04:22 2012 -0500 - make pid file default to cwd unless specified - better log output for the building - fixme notes >--------------------------------------------------------------- backend/dispatcher.py | 7 ++++++- copr-be.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index f96bdb1..16e0825 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -191,6 +191,10 @@ class Worker(multiprocessing.Process): # parse the job json into our info job = self.parse_job(jobfile) + # FIXME + # this is our best place to sanity check the job before starting + # up any longer process + job.jobfile = jobfile # spin up our build instance @@ -227,7 +231,7 @@ class Worker(multiprocessing.Process): # start the build - most importantly license checks. - self.callback.log('Calling mockremote with builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) + self.callback.log('Starting build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: chrootlogfile = chroot_destdir + '/mockremote.log' mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, @@ -239,6 +243,7 @@ class Worker(multiprocessing.Process): # record and break self.callback.log('%s - %s' % (ip, e)) status = 0 # failure + self.callback.log('Finished build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) job.ended_on = time.time() job.status = status diff --git a/copr-be.py b/copr-be.py index d98feea..5b35905 100644 --- a/copr-be.py +++ b/copr-be.py @@ -182,7 +182,7 @@ def parse_args(args): help="config file to use for copr-be run") parser.add_option('-d','--daemonize', default=False, dest='daemonize', action='store_true', help="daemonize or not") - parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', + parser.add_option('-p', '--pidfile', default='copr-be.pid', dest='pidfile', help="pid file to use for copr-be if daemonized") parser.add_option('-x', '--exit', default=False, dest='exit_on_worker', action='store_true', help="exit on worker failure")

1 0

[copr] skvidal-backend: spaces in rsync command for name of copr :-/ (858a533)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 858a5331b1902764f7ee9a3838c7c1a2c8cfe10a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:47:03 2012 -0500 spaces in rsync command for name of copr :-/ >--------------------------------------------------------------- backend/mockremote.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index 7698de9..3d39f3d 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -329,6 +329,7 @@ class Builder(object): success = False rpd = self._get_remote_pkg_dir(pkg) + destdir = "'" + destdir.replace("'", "'\\''") + "'" # make spaces work w/our rsync command below :( # build rsync command line from the above remote_src = '%s@%s:%s' % (self.username, self.hostname, rpd) ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"

1 0

[copr] skvidal-backend: still need something like a useful output when it does break (58c0441)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 58c0441ccecf4694cc45a82f43fcf372a9090528 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:14:14 2012 -0500 still need something like a useful output when it does break >--------------------------------------------------------------- copr-be.py | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diff --git a/copr-be.py b/copr-be.py index 3be1828..d98feea 100644 --- a/copr-be.py +++ b/copr-be.py @@ -28,7 +28,7 @@ class CoprBackend(object): raise errors.CoprBackendError, "Must specify config_file" self.config_file = config_file - self._ext_opts = ext_opts # to stow our cli options for read_conf() + self.ext_opts = ext_opts # to stow our cli options for read_conf() self.opts = self.read_conf() logdir = os.path.dirname(self.opts.logfile) @@ -79,8 +79,8 @@ class CoprBackend(object): if not opts.jobsdir or not opts.destdir: raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration" - if self._ext_opts: - for v in self._ext_opts: + if self.ext_opts: + for v in self.ext_opts: setattr(opts, v, self.ext_opts.get(v)) return opts @@ -213,8 +213,9 @@ def main(args): if 'cbe' in locals(): for w in cbe.workers: w.terminate() - - + raise + except KeyboardInterrupt, e: + pass if __name__ == '__main__': try:

1 0

[copr] skvidal-backend: - make opts behave in face of cli-opts and on config reread - make exit be a bit quieter on ctrl-c (I hope) - prepare for a case where our builders are not dynamically created - obey pyflakes and pychecker suggestions (15066b2)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 15066b2d6c9c9a00994d2b7be1f13612466367a7 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:08:52 2012 -0500 - make opts behave in face of cli-opts and on config reread - make exit be a bit quieter on ctrl-c (I hope) - prepare for a case where our builders are not dynamically created - obey pyflakes and pychecker suggestions >--------------------------------------------------------------- backend/dispatcher.py | 19 +++++++++++-------- copr-be.py | 20 +++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 434e671..f96bdb1 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -95,6 +95,7 @@ class Worker(multiprocessing.Process): self.opts = opts self.kill_received = False self.callback = callback + self.create = create if not self.callback: self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num self.callback = WorkerCallback(logfile = self.logfile) @@ -193,14 +194,15 @@ class Worker(multiprocessing.Process): job.jobfile = jobfile # spin up our build instance - try: - ip = self.spawn_instance() - if not ip: - raise errors.CoprWorkerError, "No IP found from creating instance" + if self.create: + try: + ip = self.spawn_instance() + if not ip: + raise errors.CoprWorkerError, "No IP found from creating instance" - except ansible.errors.AnsibleError, e: - self.callback.log('failure to setup instance: %s' % e) - raise + except ansible.errors.AnsibleError, e: + self.callback.log('failure to setup instance: %s' % e) + raise status = 1 job.started_on = time.time() @@ -243,5 +245,6 @@ class Worker(multiprocessing.Process): self.return_results(job) self.callback.log('worker finished build: %s' % ip) # clean up the instance - self.terminate_instance(ip) + if self.create: + self.terminate_instance(ip) diff --git a/copr-be.py b/copr-be.py index a3e372a..3be1828 100644 --- a/copr-be.py +++ b/copr-be.py @@ -20,7 +20,7 @@ def _get_conf(cp, section, option, default): class CoprBackend(object): - def __init__(self, config_file=None): + def __init__(self, config_file=None, ext_opts=None): # read in config file # put all the config items into a single self.opts bunch @@ -28,6 +28,7 @@ class CoprBackend(object): raise errors.CoprBackendError, "Must specify config_file" self.config_file = config_file + self._ext_opts = ext_opts # to stow our cli options for read_conf() self.opts = self.read_conf() logdir = os.path.dirname(self.opts.logfile) @@ -78,6 +79,9 @@ class CoprBackend(object): if not opts.jobsdir or not opts.destdir: raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration" + if self._ext_opts: + for v in self._ext_opts: + setattr(opts, v, self.ext_opts.get(v)) return opts @@ -188,17 +192,19 @@ def parse_args(args): print "No config file found at: %s" % opts.config_file sys.exit(1) - return opts,args + ret_opts = Bunch() + for o in ('daemonize', 'exit_on_worker', 'pidfile', 'config_file'): + setattr(ret_opts, o, getattr(opts, o)) + + return ret_opts def main(args): - opts,args = parse_args(args) + opts = parse_args(args) try: - cbe = CoprBackend(opts.config_file) - cbe.opts.daemonize = opts.daemonize # just so we have it on hand - cbe.opts.exit_on_worker = opts.exit_on_worker + cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) cbe.run() @@ -207,7 +213,7 @@ def main(args): if 'cbe' in locals(): for w in cbe.workers: w.terminate() - raise + if __name__ == '__main__':

1 0

[copr] skvidal-backend: - check daemonize before daemonizing - output time in a more human readable format (4f0bd84)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 4f0bd842fd295465eefb1037f7458ff9b620faae Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:31:38 2012 -0500 - check daemonize before daemonizing - output time in a more human readable format >--------------------------------------------------------------- backend/dispatcher.py | 2 +- copr-be.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index e1a47dc..434e671 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -74,7 +74,7 @@ class WorkerCallback(object): if not self.logfile: return - now = time.time() + now = time.strftime('%F %T') try: open(self.logfile, 'a').write(str(now) + ':' + msg + '\n') except (IOError, OSError), e: diff --git a/copr-be.py b/copr-be.py index fc03430..a3e372a 100644 --- a/copr-be.py +++ b/copr-be.py @@ -82,7 +82,7 @@ class CoprBackend(object): def log(self, msg): - now = time.time() + now = time.strftime('%F %T') output = str(now) + ':' + msg if not self.opts.daemonize: print output @@ -199,7 +199,8 @@ def main(args): cbe = CoprBackend(opts.config_file) cbe.opts.daemonize = opts.daemonize # just so we have it on hand cbe.opts.exit_on_worker = opts.exit_on_worker - daemonize(opts.pidfile) + if opts.daemonize: + daemonize(opts.pidfile) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: odd that pyflakes didn't catch a comma missing (c7f2dc7)
by Seth Vidal 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit c7f2dc73ab31b1f904c3bca2eb44c36738237e4e Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:25:23 2012 -0500 odd that pyflakes didn't catch a comma missing >--------------------------------------------------------------- copr-be.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 4fd48af..fc03430 100644 --- a/copr-be.py +++ b/copr-be.py @@ -176,7 +176,7 @@ def parse_args(args): parser = optparse.OptionParser('\ncopr-be [options]') parser.add_option('-c', '--config', default='/etc/copr-be.conf', dest='config_file', help="config file to use for copr-be run") - parser.add_option('-d' '--daemonize', default=False, dest='daemonize', + parser.add_option('-d','--daemonize', default=False, dest='daemonize', action='store_true', help="daemonize or not") parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', help="pid file to use for copr-be if daemonized")

1 0

[copr] skvidal-backend: - add option parsing for config file, exit_on_worker, daemonize and pidfile - try out daemonize support (2840524)
by Seth Vidal 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 28405241f3b747ddae19f143ff486c86431cd20a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:22:57 2012 -0500 - add option parsing for config file, exit_on_worker, daemonize and pidfile - try out daemonize support >--------------------------------------------------------------- copr-be.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 75 insertions(+), 6 deletions(-) diff --git a/copr-be.py b/copr-be.py index 73ab70a..4fd48af 100644 --- a/copr-be.py +++ b/copr-be.py @@ -10,6 +10,7 @@ from backend.dispatcher import Worker from backend import errors from bunch import Bunch import ConfigParser +import optparse def _get_conf(cp, section, option, default): """to make returning items from config parser less irritating""" @@ -59,6 +60,8 @@ class CoprBackend(object): opts.terminate_playbook = _get_conf(cp,'backend','terminate_playbook', '/etc/copr/terminate_playbook.yml') opts.jobsdir = _get_conf(cp, 'backend', 'jobsdir', None) opts.destdir = _get_conf(cp, 'backend', 'destdir', None) + opts.daemonize = _get_conf(cp, 'backend', 'daemonize', True) + opts.exit_on_worker = _get_conf(cp, 'backend', 'exit_on_worker', False) opts.sleeptime = int(_get_conf(cp, 'backend', 'sleeptime', 10)) opts.num_workers = int(_get_conf(cp, 'backend', 'num_workers', 8)) opts.timeout = int(_get_conf(cp, 'builder', 'timeout', 1800)) @@ -80,8 +83,12 @@ class CoprBackend(object): def log(self, msg): now = time.time() + output = str(now) + ':' + msg + if not self.opts.daemonize: + print output + try: - open(self.logfile, 'a').write(str(now) + ':' + msg + '\n') + open(self.logfile, 'a').write(output + '\n') except (IOError, OSError), e: print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e)) @@ -120,17 +127,79 @@ class CoprBackend(object): # check for dead workers and abort for w in self.workers: if not w.is_alive(): - raise errors.CoprBackendError, "Worker died unexpectedly" + self.log('Worker %d died unexpectedly' % w.worker_num) + if self.opts.exit_on_worker: + raise errors.CoprBackendError, "Worker died unexpectedly, exiting" + time.sleep(self.opts.sleeptime) +# lifted from certmaster +def daemonize(pidfile=None): + """ + Daemonize this process with the UNIX double-fork trick. + Writes the new PID to the provided file name if not None. + """ -def main(args): - if len(args) < 1 or not os.path.exists(args[0]): - print 'Must pass in config file' + pid = os.fork() + if pid > 0: + sys.exit(0) + os.chdir("/") + os.setsid() + os.umask(077) + pid = os.fork() + + os.close(0) + os.close(1) + os.close(2) + + # The standard I/O file descriptors are redirected to /dev/null by default. + if (hasattr(os, "devnull")): + REDIRECT_TO = os.devnull + else: + REDIRECT_TO = "/dev/null" + + # based on http://code.activestate.com/recipes/278731/ + os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) + + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) # standard error (2) + + + + if pid > 0: + if pidfile is not None: + open(pidfile, "w").write(str(pid)) + sys.exit(0) + +def parse_args(args): + parser = optparse.OptionParser('\ncopr-be [options]') + parser.add_option('-c', '--config', default='/etc/copr-be.conf', dest='config_file', + help="config file to use for copr-be run") + parser.add_option('-d' '--daemonize', default=False, dest='daemonize', + action='store_true', help="daemonize or not") + parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', + help="pid file to use for copr-be if daemonized") + parser.add_option('-x', '--exit', default=False, dest='exit_on_worker', + action='store_true', help="exit on worker failure") + + opts, args = parser.parse_args(args) + if not os.path.exists(opts.config_file): + print "No config file found at: %s" % opts.config_file sys.exit(1) + + return opts,args + + + +def main(args): + opts,args = parse_args(args) + try: - cbe = CoprBackend(args[0]) + cbe = CoprBackend(opts.config_file) + cbe.opts.daemonize = opts.daemonize # just so we have it on hand + cbe.opts.exit_on_worker = opts.exit_on_worker + daemonize(opts.pidfile) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: touch up the log outputs to make it easier to know what to do (8fc0c3e)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 8fc0c3e6a2eb3cd253ccdc75f4c2c05743e3590d Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 01:37:24 2012 -0500 touch up the log outputs to make it easier to know what to do >--------------------------------------------------------------- backend/dispatcher.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index eff1f27..e1a47dc 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -102,7 +102,7 @@ class Worker(multiprocessing.Process): if ip: self.callback.log('creating worker: %s' % ip) else: - self.callback.log('creating worker: with dynamic ip') + self.callback.log('creating worker: dynamic ip') def spawn_instance(self): """call the spawn playbook to startup/provision a building instance""" @@ -130,7 +130,9 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) - self.callback.log(play.stats.summarize('localhost')) + self.callback.log(str(play.stats.summarize('localhost'))) + self.callback.log('Test spawn_instance playbook manually') + return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: localhost as string to get back debug on heisenbug (c881da2)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit c881da21fc3f7fa723610919f627411b25e1a8e2 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 00:59:54 2012 -0500 localhost as string to get back debug on heisenbug >--------------------------------------------------------------- backend/dispatcher.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 796a9b6..eff1f27 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -130,7 +130,7 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) - self.callback.log(play.stats.summarize(localhost)) + self.callback.log(play.stats.summarize('localhost')) return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: when we're dealing with no repos specified - make sure we don't treat '' as a repo (a1349ac)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit a1349acfffefce3ee6bd56b1eac12312e619ef1a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 00:56:57 2012 -0500 when we're dealing with no repos specified - make sure we don't treat '' as a repo >--------------------------------------------------------------- backend/dispatcher.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index a9427aa..796a9b6 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -154,7 +154,7 @@ class Worker(multiprocessing.Process): build = d['builds'][0] jobdata = Bunch() jobdata.pkgs = build['pkgs'].split(' ') - jobdata.repos = build['repos'].split(' ') + jobdata.repos = [r for r in build['repos'].split(' ') if r.strip() ] jobdata.chroots = build['chroots'].split(' ') jobdata.memory_reqs = build['memory_reqs'] jobdata.timeout = build['timeout'] @@ -223,7 +223,7 @@ class Worker(multiprocessing.Process): # start the build - most importantly license checks. - self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) + self.callback.log('Calling mockremote with builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: chrootlogfile = chroot_destdir + '/mockremote.log' mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,

1 0

[copr] skvidal-backend: more debug output yay! (f568f12)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit f568f120f4dddf1fdb1818db9664d36efcded46b Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:55:14 2012 -0500 more debug output yay! >--------------------------------------------------------------- backend/dispatcher.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 4afd9f9..a9427aa 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -130,6 +130,7 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) + self.callback.log(play.stats.summarize(localhost)) return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: dear world in the future - read your own code (16acf7c)
by skvidal＠fedorahosted.org 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 16acf7ce04ae4e242691c91aa8ca10fbb289ee2a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:44:17 2012 -0500 dear world in the future - read your own code >--------------------------------------------------------------- backend/dispatcher.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index bf80e28..4afd9f9 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -228,7 +228,7 @@ class Worker(multiprocessing.Process): mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, destdir=job.destdir, chroot=chroot, cont=True, recurse=True, repos=job.repos, - callback=mockremote.CLiLogCallBack(quiet=True,logfn=chrootlogfile)) + callback=mockremote.CliLogCallBack(quiet=True,logfn=chrootlogfile)) mr.build_pkgs(job.pkgs) except mockremote.MockRemoteError, e: # record and break

1 0

[copr] skvidal-backend: more logging in error conditions having to do with spawning instances (31ab388)
by skvidal＠fedorahosted.org 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 31ab388e827dbe9a279a8bb8eac76ff1acb60212 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:37:43 2012 -0500 more logging in error conditions having to do with spawning instances >--------------------------------------------------------------- backend/dispatcher.py | 9 ++++++++- copr-be.py | 1 + 2 files changed, 9 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 225b75d..bf80e28 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -99,7 +99,10 @@ class Worker(multiprocessing.Process): self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num self.callback = WorkerCallback(logfile = self.logfile) - self.callback.log('creating worker: %s' % ip) + if ip: + self.callback.log('creating worker: %s' % ip) + else: + self.callback.log('creating worker: with dynamic ip') def spawn_instance(self): """call the spawn playbook to startup/provision a building instance""" @@ -123,6 +126,10 @@ class Worker(multiprocessing.Process): if i =='localhost': continue return i + + # if we get here we're in trouble + self.callback.log('No IP back from spawn_instance - dumping cache output') + self.callback.log(str(play.SETUP_CACHE)) return None def terminate_instance(self,ip): diff --git a/copr-be.py b/copr-be.py index 2cc0fba..73ab70a 100644 --- a/copr-be.py +++ b/copr-be.py @@ -117,6 +117,7 @@ class CoprBackend(object): # #insert a poison pill? Kill after something? I dunno. # FIXME - if a worker bombs out - we need to check them # and startup a new one if it happens + # check for dead workers and abort for w in self.workers: if not w.is_alive(): raise errors.CoprBackendError, "Worker died unexpectedly"

1 0

[copr] skvidal-backend: if any worker dies and we don't know why - raise an exception to kill us off b/c until we know all of the reasons why I'd rather just stop. (6159878)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 6159878f050c4739d84f32a8558ab36c1230298f Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:23:10 2012 -0500 if any worker dies and we don't know why - raise an exception to kill us off b/c until we know all of the reasons why I'd rather just stop. >--------------------------------------------------------------- copr-be.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/copr-be.py b/copr-be.py index 0d20e20..2cc0fba 100644 --- a/copr-be.py +++ b/copr-be.py @@ -117,6 +117,9 @@ class CoprBackend(object): # #insert a poison pill? Kill after something? I dunno. # FIXME - if a worker bombs out - we need to check them # and startup a new one if it happens + for w in self.workers: + if not w.is_alive(): + raise errors.CoprBackendError, "Worker died unexpectedly" time.sleep(self.opts.sleeptime)

1 0

[copr] skvidal-backend: typo (7f1ac35)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 7f1ac35229cb5589c6df41a0641526a0d303fe06 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:05:24 2012 -0500 typo >--------------------------------------------------------------- backend/dispatcher.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 411f95c..225b75d 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -217,7 +217,7 @@ class Worker(multiprocessing.Process): self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: - chrootlogfile = chroot_destdir + '/' + mockremote.log + chrootlogfile = chroot_destdir + '/mockremote.log' mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, destdir=job.destdir, chroot=chroot, cont=True, recurse=True, repos=job.repos,

1 0

[copr] skvidal-backend: - output what mockremote is seeing to the chroot log for the pkg - move around where we need to add check against pkgs (0f2031c)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 0f2031cdcce6c5ddccdf7d2b427453db35f7dc77 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:01:00 2012 -0500 - output what mockremote is seeing to the chroot log for the pkg - move around where we need to add check against pkgs >--------------------------------------------------------------- backend/dispatcher.py | 18 ++++++++++++++---- backend/mockremote.py | 4 ---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 286cffd..411f95c 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -196,22 +196,32 @@ class Worker(multiprocessing.Process): job.started_on = time.time() for chroot in job.chroots: + chroot_destdir = job.destdir + '/' + chroot # setup our target dir locally - if not os.path.exists(job.destdir): + if not os.path.exists(chroot_destdir): try: - os.makedirs(job.destdir, mode=0755) + os.makedirs(chroot_destdir, mode=0755) except (OSError, IOError), e: - msg = "Could not make results dir for job: %s - %s" % (job.destdir, str(e)) + msg = "Could not make results dir for job: %s - %s" % (chroot_destdir, str(e)) self.callback.log(msg) status = 0 continue + # FIXME + # need a plugin hook or some mechanism to check random + # info about the pkgs + # this should use ansible to download the pkg on the remote system + # and run a series of checks on the package before we + # start the build - most importantly license checks. + + self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: + chrootlogfile = chroot_destdir + '/' + mockremote.log mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, destdir=job.destdir, chroot=chroot, cont=True, recurse=True, repos=job.repos, - callback=mockremote.DefaultCallBack(quiet=True,logfn=self.logfile)) + callback=mockremote.CLiLogCallBack(quiet=True,logfn=chrootlogfile)) mr.build_pkgs(job.pkgs) except mockremote.MockRemoteError, e: # record and break diff --git a/backend/mockremote.py b/backend/mockremote.py index a2a5f13..7698de9 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -492,10 +492,6 @@ class MockRemote(object): r_log.write(b_err) r_log.close() - # FIXME - # need a plugin hook or some mechanism to check random - # info about the pkgs - # checking where to stick stuff if not b_status:

1 0

[copr] skvidal-backend: output whatever mockremote sees a little more obviously (a70a73c)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit a70a73c958cd1a9831146c91106b486bfa0e2b08 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 15:23:11 2012 -0500 output whatever mockremote sees a little more obviously >--------------------------------------------------------------- backend/mockremote.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index a5d39d5..a2a5f13 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -486,10 +486,10 @@ class MockRemote(object): os.makedirs(self.destdir + '/' + self.chroot) r_log = open(self.destdir + '/' + self.chroot + '/mockchain.log', 'a') r_log.write('%s\n' % pkg) - r_log.write('stdout\n') r_log.write(b_out) - r_log.write('stderr\n') - r_log.write(b_err) + if b_err: + r_log.write('\nstderr\n') + r_log.write(b_err) r_log.close() # FIXME

1 0

[copr] skvidal-backend: typo . instead of _ (2884bdb)
by Seth Vidal 30 Nov '12

30 Nov '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 2884bdbe71787f99faa1948f1b4a6dfea268b819 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Sat Dec 1 00:34:59 2012 -0500 typo . instead of _ >--------------------------------------------------------------- backend/mockremote.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index 373f237..a5d39d5 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -488,9 +488,9 @@ class MockRemote(object): r_log.write('%s\n' % pkg) r_log.write('stdout\n') r_log.write(b_out) - r.log.write('stderr\n') - r.log.write(b_err) - r.log.close() + r_log.write('stderr\n') + r_log.write(b_err) + r_log.close() # FIXME # need a plugin hook or some mechanism to check random

1 0

[copr] skvidal-backend: make sure out results dir is there (a58fc08)
by Seth Vidal 30 Nov '12

30 Nov '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit a58fc088a29122be6b6d342f4fa9edb1bba591ba Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Sat Dec 1 00:30:49 2012 -0500 make sure out results dir is there >--------------------------------------------------------------- backend/mockremote.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index d7d0eed..373f237 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -482,6 +482,8 @@ class MockRemote(object): self.callback.end_download(pkg) # write out whatever came from the builder call into the destdir/chroot + if not os.path.exists(self.destdir + '/' + self.chroot): + os.makedirs(self.destdir + '/' + self.chroot) r_log = open(self.destdir + '/' + self.chroot + '/mockchain.log', 'a') r_log.write('%s\n' % pkg) r_log.write('stdout\n')

1 0

[copr] skvidal-backend: need a bit more logging detail to see what's breaking and why (d29ca18)
by skvidal＠fedorahosted.org 30 Nov '12

30 Nov '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit d29ca180260fc7734eec05f31d0fe7ffbd855898 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Sat Dec 1 00:24:46 2012 -0500 need a bit more logging detail to see what's breaking and why >--------------------------------------------------------------- backend/mockremote.py | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index 5bfb1f2..d7d0eed 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -481,6 +481,14 @@ class MockRemote(object): self.callback.error(msg) self.callback.end_download(pkg) + # write out whatever came from the builder call into the destdir/chroot + r_log = open(self.destdir + '/' + self.chroot + '/mockchain.log', 'a') + r_log.write('%s\n' % pkg) + r_log.write('stdout\n') + r_log.write(b_out) + r.log.write('stderr\n') + r.log.write(b_err) + r.log.close() # FIXME # need a plugin hook or some mechanism to check random

1 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

copr-devel December 2012