Hey all,
this patch solves ticket #241. It adds options in autoqa.conf to turn on/off optin, notifications and results e-mails. There are options containing urls of koji and bodhi instances that will be used.
Also there are some alterations in bodhi_utils, there is only one creation of BodhiClient instance and one config files reading to simplify the code.
You can find the patch in mkrizek-staging-ready branch.
--- diff --git a/autoqa b/autoqa index fd41503..190e63b 100755 --- a/autoqa +++ b/autoqa @@ -40,12 +40,14 @@ conf = { 'testdir': '/usr/share/autotest/client/site_tests', 'hookdir': '/usr/share/autoqa', 'notification_email': '', + 'send_notification_email': 'false', 'autotest_server': socket.gethostname(), }
cfg_parser = SingleConfigParser() cfg_parser.read(cfgfile) -conf = cfg_parser.get_section('general', conf) # used by prep_controlfile +conf = cfg_parser.get_section('general', conf) +conf = cfg_parser.get_section('notifications', conf) # we don't need to catch errors here, bcz we want autoqa to crash for invalid config
def prep_controlfile(controlfile, extradata): @@ -283,7 +285,10 @@ for test in testlist:
for arch in test_vars[test]['archs']: testname='%s:%s.%s' % (hookname, test, arch) - email = conf['notification_email'] + if conf['send_notification_email']: + email = conf['notification_email'] + else: + email = ''
if run_local: retval = run_test_locally(control, name=testname, diff --git a/autoqa.conf b/autoqa.conf index 69278fe..886536e 100644 --- a/autoqa.conf +++ b/autoqa.conf @@ -6,29 +6,62 @@ hookdir = /usr/share/autoqa # Path containing the actual autoqa tests to be run testdir = /usr/share/autotest/client/site_tests -# comma-separated email addresses that will get notification of test completion -# (leave blank to disable) -notification_email = # Set to "true" if you want autoqa to always run tests locally using 'autotest'. # Default is to schedule jobs with the autotest server using 'atest'. # This is the same as using 'autoqa --local'. -#local = true +local = false +# Hostname of this autotest server. Used for constructing URLs for accessing +# test results. By default current hostname is used. +autotest_server =
-# The 'test' section is used for systemwide configuration of autoqa tests -[test] -# comma-separated list of email addresses that will receive full test results -# (leave blank to disable) -result_email = -# email address for From: line of emails sent by tests -mail_from = autoqa@fedoraproject.org -# hostname or hostname:port of smtp server / mailhub to use for sending email +# The 'notifications' section controls which notifications about which events +# you want to send +[notifications] +# Sender email address used for all emails sent from AutoQA. +mail_from = +# Hostname or hostname:port of SMTP server used for sending emails. +# Please note that the emails are sent from the autotest clients, not the +# server. smtpserver = localhost
-[bodhi] -# If "true", test results (for tests utilizing this feature) will be sent -# as comments to Fedora Update System (Bodhi). This requires that you have -# Bodhi credentials filled in in fas.conf. +# Comma-separated list of email addresses that will receive full test results +# of every completed test. +result_email = +# If "true", result email will be sent. Otherwise "false". +send_result_email = false + +# Opt-in emails are sent (for selected tests) to the maintainers of a package +# that has just been tested and the maintainer opted in. Read more at: +# http://jlaska.wordpress.com/2010/06/01/fedora-package-maintainers-want-test-... +# If "true", opt-in email will be sent. Otherwise "false". +send_optin_email = false + +# Comma-separated email addresses that will get notification of every job +# completion or every job state change as defined in 'notify_email_statuses' +# in ~autotest/global_config.ini. +# Important: These emails are handled and sent by autotest server, the +# 'mail_from' and 'smtpserver' options don't apply for it. +# Read more at http://autotest.kernel.org/wiki/GlobalConfig (search for +# 'notify_email'). +notification_email = +# If "true", notification email will be sent. Otherwise "false". +send_notification_email = false + +# If "true", test results (for selected tests) will be sent as comments to +# Fedora Update System (Bodhi). This requires that you have Bodhi credentials +# filled in in fas.conf. send_bodhi_comments = false -# how long (minutes) should we wait before posting the same comment to bodhi -# by default 3 days (3*24*60 = 4320) +# How long (in minutes) should we wait before allowing consequent test to +# re-post a 'FAILED' comment into Bodhi once again. +# By default 3 days (3*24*60 = 4320). bodhi_posting_comments_span = 4320 + +# The 'resources' section specify access details to various external services +[resources] +# URL of Koji instance used for querying about new builds +koji_url = http://koji.fedoraproject.org/kojihub +# URL of repository of all the RPM packages built in Koji +pkg_url = http://koji.fedoraproject.org/packages +# URL of Bodhi instance used for communication about package updates +# (leave blank for default) +bodhi_server = diff --git a/hooks/post-koji-build/watch-koji-builds.py b/hooks/post-koji-build/watch-koji-builds.py index 4cc2d7f..57f0aee 100755 --- a/hooks/post-koji-build/watch-koji-builds.py +++ b/hooks/post-koji-build/watch-koji-builds.py @@ -41,9 +41,6 @@ try: except OSError, e: if e.errno != 17: # already exists raise -# XXX configparser? /etc/koji.conf, section 'koji', item 'server' -# alternately: read from e.g. /etc/autoqa/autoqa.conf -kojiserver = 'http://koji.fedoraproject.org/kojihub'
def get_prevtime(): '''Returns the prevtime to use when looking for new builds. Value will be @@ -126,7 +123,7 @@ tags for new builds and kick off tests when new builds/packages are found.')
# Set up the koji connection kojiopts = {} # Possible items: user, password, debug_xmlrpc, debug.. - session = koji_utils.SimpleKojiClientSession(kojiserver, kojiopts) + session = koji_utils.SimpleKojiClientSession(kojiopts) untagged_builds = [] tagged_builds = [] exit_status = 0 diff --git a/lib/python/bodhi_utils.py b/lib/python/bodhi_utils.py index 94e1a7d..95d18a5 100644 --- a/lib/python/bodhi_utils.py +++ b/lib/python/bodhi_utils.py @@ -28,6 +28,32 @@ from datetime import datetime from util import SingleConfigParser, getbool import ConfigParser
+try: + autoqa_conf = SingleConfigParser() + autoqa_conf.read_single(['autoqa.conf', '/etc/autoqa/autoqa.conf']) + + server = autoqa_conf.get('resources', 'bodhi_server') +except ConfigParser.Error, e: + server = '' + +try: + fas_conf = SingleConfigParser() + fas_conf.read_single(['fas.conf', '/etc/autoqa/fas.conf']) + fas = fas_conf.get_section('fas') + user = fas['username'] + pswd = fas['password'] + if not user or not pswd: + raise IOError +except (IOError, ConfigParser.Error), e: + # if there's no fas.conf or it's incomplete, we just log in anonymously. + user = '' + pswd = '' + +if server: + bodhi = fedora.client.bodhi.BodhiClient(base_url=server, username=user, password=pswd) +else: + bodhi = fedora.client.bodhi.BodhiClient(username=user, password=pswd) + def bodhitime(timestamp): '''Convert timestamp (seconds since Epoch, assumed to be local time) to a Bodhi-approved time string ('%Y-%m-%d %H:%M:%S')''' @@ -40,7 +66,6 @@ def parse_bodhitime(bodhitimestr):
def bodhi_list(params, limit=100): '''Perform a bodhi 'list' method call, with pagination handling''' - bodhi = fedora.client.bodhi.BodhiClient() params['tg_paginate_limit'] = limit params['tg_paginate_no'] = 1
@@ -136,7 +161,7 @@ def _is_bodhi_testresult_needed(old_result, comment_time, result, time_span):
return True
-def bodhi_post_testresult(update, testname, result, url, config, arch = 'noarch', karma = 0): +def bodhi_post_testresult(update, testname, result, url, arch = 'noarch', karma = 0): '''Post comment and karma to bodhi
Args: @@ -144,7 +169,6 @@ def bodhi_post_testresult(update, testname, result, url, config, arch = 'noarch' testname -- the name of the test result -- the result of the test url -- url of the result of the test - config -- autoqa config arch -- tested architecture (default 'noarch') karma -- karma points (default 0)
@@ -153,37 +177,25 @@ def bodhi_post_testresult(update, testname, result, url, config, arch = 'noarch' posted (either posting is turned off or comment was already posted), False otherwise. ''' + + # TODO when new bodhi releases, add update identification by UPDATEID support + err_msg = 'Could not post a comment to bodhi' - conffiles = ['fas.conf', '/etc/autoqa/fas.conf'] - try: - cfg_parser = SingleConfigParser() - cfg_parser.read_single(conffiles) - fas = cfg_parser.get_section('fas') - except (IOError, ConfigParser.Error), e: - print >> sys.stderr, "ERROR: Couldn't read none of config files: %s" % conffiles - print >> sys.stderr, e - return False
if not update or not testname or not result or url == None: sys.stderr.write('Incomplete arguments!\n%s\n' % err_msg) return False
try: - if not getbool(config.get('bodhi', 'send_bodhi_comments')): - print 'Sending bodhi comments is turned off. Test result will NOT be sent.' - return True - except KeyError: + if not getbool(autoqa_conf.get('notifications', 'send_bodhi_comments')): + raise ValueError + except ValueError: print 'Sending bodhi comments is turned off. Test result will NOT be sent.' - # option missing -> it's false, do not send it (but return True since + # it's either False or not a bool -> it's false, do not send it (but return True since # it's intentional, not an error) return True
- try: - user = fas['username'] - pswd = fas['password'] - if not user or not pswd: - raise KeyError - except KeyError: + if not user or not pswd: sys.stderr.write('Conf file containing FAS credentials is incomplete!\n%s\n' % err_msg) return False
@@ -191,14 +203,12 @@ def bodhi_post_testresult(update, testname, result, url, config, arch = 'noarch' % (testname, result, arch, url) try: (old_result, comment_time) = _bodhi_already_commented(update, user, testname, arch) - time_span = int(config.get('bodhi', 'bodhi_posting_comments_span')) + time_span = int(autoqa_conf.get('notifications', 'bodhi_posting_comments_span'))
if not _is_bodhi_testresult_needed(old_result, comment_time, result, time_span): print 'The test result already posted to bodhi.' return True
- bodhi = fedora.client.BodhiClient(username=user, password=pswd) - if not bodhi.comment(update, comment, karma): sys.stderr.write('%s\n') % err_msg return False @@ -216,10 +226,7 @@ def _self_test(): Simple self test. ''' from datetime import timedelta - from ConfigParser import SafeConfigParser - cfg_parser = SafeConfigParser() - cfg_parser.read('/etc/autoqa/autoqa.conf') - time_span = int(cfg_parser.get('bodhi', 'bodhi_posting_comments_span')) + time_span = int(autoqa_conf.get('notifications', 'bodhi_posting_comments_span')) try: print '1. Test:', assert _is_bodhi_testresult_needed('PASSED', datetime.now, 'PASSED', time_span) == False diff --git a/lib/python/koji_utils.py b/lib/python/koji_utils.py index 87b32e9..648412b 100644 --- a/lib/python/koji_utils.py +++ b/lib/python/koji_utils.py @@ -24,15 +24,16 @@ import koji from repoinfo import repoinfo import rpmUtils.miscutils import sys - -# XXX fetch from /etc/koji.conf, section 'koji' -kojiurl = 'http://koji.fedoraproject.org/kojihub' -pkgurl = 'http://koji.fedoraproject.org/packages' +from autoqa.bodhi_utils import SingleConfigParser
class SimpleKojiClientSession(koji.ClientSession): '''Convenience wrapper class for interacting with koji''' - def __init__(self, server=kojiurl, opts=None): - return koji.ClientSession.__init__(self, server, opts) + def __init__(self, opts=None): + cfg_parser = SingleConfigParser() + cfg_parser.read_single(['autoqa.conf', '/etc/autoqa/autoqa.conf']) + self.server = cfg_parser.get('resources', 'koji_url') + self.pkgurl = cfg_parser.get('resources', 'pkg_url') + return koji.ClientSession.__init__(self, self.server, opts)
def list_builds_since(self, timestamp): '''Return a list of new builds since the given timestamp''' @@ -147,7 +148,7 @@ class SimpleKojiClientSession(koji.ClientSession):
def nvr_to_urls(self, nvr, arches=None, debuginfo=False): info = self.getBuild(nvr) - baseurl = '/'.join((pkgurl, info['package_name'], + baseurl = '/'.join((self.pkgurl, info['package_name'], info['version'], info['release']))
rpms = self.listRPMs(buildID=info['id'], arches=arches) @@ -162,7 +163,7 @@ class SimpleKojiClientSession(koji.ClientSession): key added with URL where to download the RPM as the value.'''
info = self.getBuild(nvr) - baseurl = '/'.join((pkgurl, info['package_name'], + baseurl = '/'.join((self.pkgurl, info['package_name'], info['version'], info['release']))
rpms = self.listRPMs(buildID=info['id']) diff --git a/lib/python/test.py b/lib/python/test.py index 29bb96c..73efc80 100644 --- a/lib/python/test.py +++ b/lib/python/test.py @@ -21,7 +21,7 @@ from autotest_lib.client.bin import test, utils from autotest_lib.client.bin.test_config import config_loader
from decorators import ExceptionCatcher -from util import make_autotest_url +from util import make_autotest_url, getbool import os, sys, traceback
class AutoQATest(test.test, object): @@ -80,7 +80,7 @@ class AutoQATest(test.test, object):
# prepare email send_to = [] - result_email = self.config.get('test','result_email') + result_email = self.config.get('notifications','result_email') if result_email and isinstance(result_email, str): send_to = result_email.split(',') try: @@ -114,12 +114,12 @@ class AutoQATest(test.test, object): log.close()
# send email - if send_to: + if getbool(self.config.get('notifications', 'send_result_email')) and send_to: utils.send_email(mail_to=send_to, - mail_from=self.config.get('test','mail_from'), - mail_server=self.config.get('test','smtpserver'), - subject=self.summary, - body = mail_body) + mail_from=self.config.get('notifications','mail_from'), + mail_server=self.config.get('notifications','smtpserver'), + subject=self.summary, + body = mail_body)
def _convert_list_variables(self): ''' diff --git a/tests/initscripts/initscripts.py b/tests/initscripts/initscripts.py index 8e35e35..097fbc8 100644 --- a/tests/initscripts/initscripts.py +++ b/tests/initscripts/initscripts.py @@ -43,7 +43,7 @@ class initscripts(AutoQATest): f = open("/etc/yum.repos.d/beakerlib.repo", "w") f.write("[beakerlib-testing]\nname=Testing repo for BeakerLib\nbaseurl=http://afri.fedorapeople.org/beakerlib/%5Cnenabled=1%5Cngpgcheck=0%5Cn") f.close() - utils.system('yum -y install beakerlib') + utils.system('yum -y install beakerlib')
@ExceptionCatcher() def initialize(self, config, **kwargs): @@ -174,7 +174,9 @@ class initscripts(AutoQATest):
# email results to mailing list and to pkg owner if they optin repo = repoinfo.getrepo_by_tag(kojitag) - if repo is not None and autoqa.util.check_opt_in(name, repo['collection_name']): + send_optin_email = getbool(self.config.get('notifications', 'send_optin_email')) + if repo is not None and send_optin_email and \ + autoqa.util.check_opt_in(pkg_name, repo['collection_name']): #FIXME - hardcoded partial email address here - obviously sub-par self.mail_to.append('%s-owner@fedoraproject.org' % name)
diff --git a/tests/rpmguard/rpmguard.py b/tests/rpmguard/rpmguard.py index bf973f0..4a20d53 100644 --- a/tests/rpmguard/rpmguard.py +++ b/tests/rpmguard/rpmguard.py @@ -82,7 +82,9 @@ class rpmguard(AutoQATest): # email results to mailing list and to pkg owner if they optin repo = repoinfo.getrepo_by_tag(kojitag) pkg_name = rpmUtils.miscutils.splitFilename(envr + '.noarch')[0] - if repo is not None and autoqa.util.check_opt_in(pkg_name, repo['collection_name']): + send_optin_email = getbool(self.config.get('notifications', 'send_optin_email')) + if repo is not None and send_optin_email and \ + autoqa.util.check_opt_in(pkg_name, repo['collection_name']): #FIXME - hardcoded partial email address here - obviously sub-par self.mail_to.append('%s-owner@fedoraproject.org' % pkg_name)
diff --git a/tests/rpmlint/rpmlint.py b/tests/rpmlint/rpmlint.py index 1c06a8b..cac8a48 100644 --- a/tests/rpmlint/rpmlint.py +++ b/tests/rpmlint/rpmlint.py @@ -116,7 +116,9 @@ class rpmlint(AutoQATest): # email results to mailing list and to pkg owner if they optin repo = repoinfo.getrepo_by_tag(kojitag) pkg_name = rpmUtils.miscutils.splitFilename(envr + '.noarch')[0] - if repo is not None and autoqa.util.check_opt_in(pkg_name, repo['collection_name']): + send_optin_email = getbool(self.config.get('notifications', 'send_optin_email')) + if repo is not None and send_optin_email and \ + autoqa.util.check_opt_in(pkg_name, repo['collection_name']): #FIXME - hardcoded partial email address here - obviously sub-par self.mail_to.append('%s-owner@fedoraproject.org' % pkg_name)